diff --git a/AnimeStudio.CLI/AnimeStudio.CLI.csproj b/AnimeStudio.CLI/AnimeStudio.CLI.csproj index 708f5d83..8c1e0229 100644 --- a/AnimeStudio.CLI/AnimeStudio.CLI.csproj +++ b/AnimeStudio.CLI/AnimeStudio.CLI.csproj @@ -9,8 +9,8 @@ - - + + diff --git a/AnimeStudio.GUI/AnimeStudio.GUI.csproj b/AnimeStudio.GUI/AnimeStudio.GUI.csproj index 7afe47f7..60d0c4c8 100644 --- a/AnimeStudio.GUI/AnimeStudio.GUI.csproj +++ b/AnimeStudio.GUI/AnimeStudio.GUI.csproj @@ -127,7 +127,7 @@ - + diff --git a/AnimeStudio.GUI/AssetBrowser.cs b/AnimeStudio.GUI/AssetBrowser.cs index 7e53c6f2..5efa5cbc 100644 --- a/AnimeStudio.GUI/AssetBrowser.cs +++ b/AnimeStudio.GUI/AssetBrowser.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.ComponentModel; -using System.Data; using System.Drawing; using System.Globalization; using System.IO; @@ -16,17 +15,17 @@ namespace AnimeStudio.GUI { partial class AssetBrowser : Form { - private readonly MainForm _parent; - private readonly List _assetEntries; - private readonly List _backupAssetEntries; - private readonly List _firstAssetEntries; - private readonly List _secondAssetEntries; + private readonly List _assetEntries; + private readonly List _backupAssetEntries; private readonly Dictionary _filters; + private readonly List _firstAssetEntries; + private readonly MainForm _parent; + private readonly List _secondAssetEntries; + private DataGridViewColumn _sortedColumn; - private SortOrder _sortOrder; - private DataGridViewColumn _sortedColumn; - private List types = new(); + private SortOrder _sortOrder; private List selectedTypes = new(); + private List types = new List(); public AssetBrowser(MainForm form) { @@ -44,7 +43,11 @@ private async void loadAssetMap_Click(object sender, EventArgs e) { loadAssetMap.Enabled = false; - var openFileDialog = new OpenFileDialog() { Multiselect = false, Filter = "MessagePack AssetMap File|*.map|JSON AssetMap File|*.json" }; + var openFileDialog = new OpenFileDialog + { + Multiselect = false, Filter = "MessagePack AssetMap File|*" + + ".map|JSON AssetMap File|*.json|MemoryPack AssetMap File|*.memory" + }; if (openFileDialog.ShowDialog(this) == DialogResult.OK) { try @@ -61,6 +64,8 @@ private async void loadAssetMap_Click(object sender, EventArgs e) _sortedColumn = null; _firstAssetEntries.Clear(); + _secondAssetEntries.Clear(); + _backupAssetEntries.Clear(); _firstAssetEntries.AddRange(ResourceMap.GetEntries()); updateDisplay(); @@ -172,12 +177,14 @@ private void bringMainToFront() _parent.TopMost = false; _parent.Focus(); } + private void clear_Click(object sender, EventArgs e) { Clear(); updateButtons(); Logger.Info($"Cleared !!"); } + private void loadSelected_Click(object sender, EventArgs e) { var files = assetDataGridView.SelectedRows.Cast() @@ -210,6 +217,7 @@ private void loadSelected_Click(object sender, EventArgs e) _parent.Invoke(() => _parent.LoadPaths(files, filePaths.ToArray())); } } + private async void exportSelected_Click(object sender, EventArgs e) { var saveFolderDialog = new OpenFolderDialog(); @@ -245,6 +253,7 @@ await Task.Run(async () => StatusStripUpdate = statusStripUpdate; } } + private void BuildAssetData(List exportableAssets, AssetEntry[] entries) { var objectAssetItemDic = new Dictionary(); @@ -284,6 +293,7 @@ private void BuildAssetData(List exportableAssets, AssetEntry[] entri exportableAssets.Clear(); exportableAssets.AddRange(matches); } + private void ProcessAssetData(Object asset, List exportableAssets, Dictionary objectAssetItemDic, List<(PPtr, string)> mihoyoBinDataNames, List<(PPtr, string)> containers) { var assetItem = new AssetItem(asset); @@ -388,6 +398,7 @@ private void FilterAssetDataGrid() assetDataGridView.RowCount = _assetEntries.Count; assetDataGridView.Refresh(); } + private void TryAddFilter(string name, string value) { Regex regex; @@ -410,6 +421,7 @@ private void TryAddFilter(string name, string value) _filters[name] = regex; } } + private void NameTextBox_KeyPress(object sender, KeyPressEventArgs e) { if (sender is TextBox textBox && e.KeyChar == (char)Keys.Enter) @@ -417,6 +429,7 @@ private void NameTextBox_KeyPress(object sender, KeyPressEventArgs e) FilterAssetDataGrid(); } } + private void ContainerTextBox_KeyPress(object sender, KeyPressEventArgs e) { if (sender is TextBox textBox && e.KeyChar == (char)Keys.Enter) @@ -424,6 +437,7 @@ private void ContainerTextBox_KeyPress(object sender, KeyPressEventArgs e) FilterAssetDataGrid(); } } + private void SourceTextBox_KeyPress(object sender, KeyPressEventArgs e) { if (sender is TextBox textBox && e.KeyChar == (char)Keys.Enter) @@ -431,6 +445,7 @@ private void SourceTextBox_KeyPress(object sender, KeyPressEventArgs e) FilterAssetDataGrid(); } } + private void PathTextBox_KeyPress(object sender, KeyPressEventArgs e) { if (sender is TextBox textBox && e.KeyChar == (char)Keys.Enter) @@ -438,6 +453,7 @@ private void PathTextBox_KeyPress(object sender, KeyPressEventArgs e) FilterAssetDataGrid(); } } + private void HashTextBox_KeyPress(object sender, KeyPressEventArgs e) { if (sender is TextBox textBox && e.KeyChar == (char)Keys.Enter) @@ -445,6 +461,7 @@ private void HashTextBox_KeyPress(object sender, KeyPressEventArgs e) FilterAssetDataGrid(); } } + private void AssetDataGridView_CellValueNeeded(object sender, DataGridViewCellValueEventArgs e) { if (_assetEntries.Count != 0 && e.RowIndex <= _assetEntries.Count) @@ -462,6 +479,7 @@ private void AssetDataGridView_CellValueNeeded(object sender, DataGridViewCellVa }; } } + private void AssetListView_ColumnHeaderMouseClick(object sender, DataGridViewCellMouseEventArgs e) { if (e.ColumnIndex <= assetDataGridView.Columns.Count) @@ -511,15 +529,22 @@ private void AssetListView_ColumnHeaderMouseClick(object sender, DataGridViewCel assetDataGridView.Refresh(); } } + private void AssetBrowser_FormClosing(object sender, FormClosingEventArgs e) { Clear(); base.OnClosing(e); } + public void Clear() { ResourceMap.Clear(); _assetEntries.Clear(); + _backupAssetEntries.Clear(); + _firstAssetEntries.Clear(); + _secondAssetEntries.Clear(); + selectedTypes.Clear(); + types.Clear(); assetDataGridView.Rows.Clear(); } @@ -699,7 +724,5 @@ private void filterSelectTypesBtn_Click(object sender, EventArgs e) popup.ShowDialog(this); } - - } } diff --git a/AnimeStudio.GUI/Components/AssetItem.cs b/AnimeStudio.GUI/Components/AssetItem.cs index aabfffcc..efeb8965 100644 --- a/AnimeStudio.GUI/Components/AssetItem.cs +++ b/AnimeStudio.GUI/Components/AssetItem.cs @@ -11,10 +11,13 @@ public class AssetItem : ListViewItem public long m_PathID; public long FullSize; public ClassIDType Type; - public string Hash; public string InfoText; public string UniqueID; public GameObjectTreeNode TreeNode; + private string hash; + private bool subItemsInitialized; + + public string Hash => hash ??= Asset.GetHash(); public AssetItem(Object asset) { @@ -25,11 +28,15 @@ public AssetItem(Object asset) TypeString = Type.ToString(); m_PathID = asset.m_PathID; FullSize = asset.byteSize; - Hash = asset.GetHash(); } public void SetSubItems() { + if (subItemsInitialized) + { + return; + } + SubItems.AddRange(new[] { Container, //Container @@ -38,6 +45,28 @@ public void SetSubItems() FullSize.ToString(), //Size Hash, //Hash }); + subItemsInitialized = true; + } + + public string GetColumnText(int column) + { + return column switch + { + 0 => Text, + 1 => Container, + 2 => TypeString, + 3 => m_PathID.ToString(), + 4 => FullSize.ToString(), + 5 => Hash, + _ => string.Empty, + }; + } + + public bool MatchesListSearch(System.Text.RegularExpressions.Regex regex) + { + return regex.IsMatch(Text) + || regex.IsMatch(Container) + || regex.IsMatch(m_PathID.ToString()); } } } diff --git a/AnimeStudio.GUI/MainForm.cs b/AnimeStudio.GUI/MainForm.cs index 946b8401..65053584 100644 --- a/AnimeStudio.GUI/MainForm.cs +++ b/AnimeStudio.GUI/MainForm.cs @@ -29,66 +29,36 @@ namespace AnimeStudio.GUI { partial class MainForm : Form { - private AssetItem lastSelectedItem; + private AboutForm aboutForm; private AssetBrowser assetBrowser; - private AboutForm aboutForm; + private FMOD.Channel channel; + private uint FMODlenms; + private float FMODVolume = 0.8f; private GameSelector gameSelector; - private UnityCNEdit unityCNEdit; private DirectBitmap imageTexture; - private string tempClipboard; + private AssetItem lastSelectedItem; - private FMOD.System system; - private FMOD.Sound sound; - private FMOD.Channel channel; + private GUILogger logger; + private FMOD.MODE loopMode = FMOD.MODE.LOOP_OFF; private FMOD.SoundGroup masterSoundGroup; - private FMOD.MODE loopMode = FMOD.MODE.LOOP_OFF; - private uint FMODlenms; - private float FMODVolume = 0.8f; - - private bool themeIsFirstLaunch = true; - - #region TexControl - private static char[] textureChannelNames = new[] { 'B', 'G', 'R', 'A' }; - private bool[] textureChannels = new[] { true, true, true, true }; - #endregion - - #region GLControl - private bool glControlLoaded; - private int mdx, mdy; - private bool lmdown, rmdown; - private int pgmID, pgmColorID, pgmBlackID; - private int attributeVertexPosition; - private int attributeNormalDirection; - private int attributeVertexColor; - private int uniformModelMatrix; - private int uniformViewMatrix; - private int uniformProjMatrix; - private int vao; - private OpenTK.Mathematics.Vector3[] vertexData; - private OpenTK.Mathematics.Vector3[] normalData; - private OpenTK.Mathematics.Vector3[] normal2Data; - private OpenTK.Mathematics.Vector4[] colorData; - private Matrix4 modelMatrixData; - private Matrix4 viewMatrixData; - private Matrix4 projMatrixData; - private int[] indiceData; - private int wireFrameMode; - private int shadeMode; - private int normalMode; - #endregion - - //asset list sorting - private int sortColumn = -1; - private bool reverseSort; //tree search private int nextGObject; - private List treeSrcResults = new List(); private string openDirectoryBackup = string.Empty; + private bool reverseSort; private string saveDirectoryBackup = string.Empty; - private GUILogger logger; + //asset list sorting + private int sortColumn = -1; + private FMOD.Sound sound; + + private FMOD.System system; + private string tempClipboard; + + private bool themeIsFirstLaunch = true; + private List treeSrcResults = new List(); + private UnityCNEdit unityCNEdit; public MainForm() { @@ -303,6 +273,7 @@ private void InitalizeOptions() } } } + private void MainForm_DragEnter(object sender, DragEventArgs e) { if (e.Data.GetDataPresent(DataFormats.FileDrop)) @@ -714,6 +685,7 @@ private void enablePreview_Check(object sender, EventArgs e) Properties.Settings.Default.enablePreview = enablePreview.Checked; Properties.Settings.Default.Save(); } + private void displayAssetInfo_Check(object sender, EventArgs e) { if (displayInfo.Checked && assetInfoLabel.Text != null) @@ -744,7 +716,9 @@ private void assetListView_RetrieveVirtualItem(object sender, RetrieveVirtualIte { if (e.ItemIndex < visibleAssets.Count) { - e.Item = visibleAssets[e.ItemIndex]; + var item = visibleAssets[e.ItemIndex]; + item.SetSubItems(); + e.Item = item; } } @@ -942,8 +916,8 @@ private void assetListView_ColumnClick(object sender, ColumnClickEventArgs e) { visibleAssets.Sort((a, b) => { - var at = a.SubItems[sortColumn].Text; - var bt = b.SubItems[sortColumn].Text; + var at = a.GetColumnText(sortColumn); + var bt = b.GetColumnText(sortColumn); return reverseSort ? bt.CompareTo(at) : at.CompareTo(bt); }); } @@ -1510,6 +1484,7 @@ private void PreviewGameObject(GameObject m_GameObject) var model = new ModelConverter(m_GameObject, options, Array.Empty()); PreviewModel(model); } + private void PreviewAnimator(Animator m_Animator) { var options = new ModelConverter.Options() @@ -1984,6 +1959,7 @@ private void toolStripMenuItem9_Click(object sender, EventArgs e) { ExportAssets(ExportFilter.Filtered, ExportType.Dump); } + private void toolStripMenuItem17_Click(object sender, EventArgs e) { ExportAssets(ExportFilter.All, ExportType.JSON); @@ -2094,10 +2070,7 @@ private void FilterAssetList() listSearch.Text = ""; } var regex = new Regex(listSearch.Text, RegexOptions.IgnoreCase); - visibleAssets = visibleAssets.FindAll( - x => regex.IsMatch(x.Text) || - regex.IsMatch(x.SubItems[1].Text) || - regex.IsMatch(x.SubItems[3].Text)); + visibleAssets = visibleAssets.FindAll(x => x.MatchesListSearch(regex)); } assetListView.VirtualListSize = visibleAssets.Count; assetListView.EndUpdate(); @@ -2173,6 +2146,7 @@ private void toolStripMenuItem15_Click(object sender, EventArgs e) { logger.ShowErrorMessage = toolStripMenuItem15.Checked; } + private async void toolStripMenuItem19_DropDownOpening(object sender, EventArgs e) { if (specifyAIVersion.Enabled && await AIVersionManager.FetchVersions()) @@ -2253,7 +2227,10 @@ private void UpdateContainers() if (!string.IsNullOrEmpty(path)) { asset.Container = path; - asset.SubItems[1].Text = path; + if (asset.SubItems.Count > 1) + { + asset.SubItems[1].Text = path; + } if (asset.Type == ClassIDType.MiHoYoBinData) { asset.Text = Path.GetFileNameWithoutExtension(path); @@ -2286,6 +2263,7 @@ private void tabControl2_SelectedIndexChanged(object sender, EventArgs e) dumpTextBox.Text = DumpAsset(lastSelectedItem.Asset); } } + private void enableResolveDependencies_CheckedChanged(object sender, EventArgs e) { Properties.Settings.Default.enableResolveDependencies = enableResolveDependencies.Checked; @@ -2293,16 +2271,19 @@ private void enableResolveDependencies_CheckedChanged(object sender, EventArgs e assetsManager.ResolveDependencies = enableResolveDependencies.Checked; } + private void allowDuplicates_CheckedChanged(object sender, EventArgs e) { Properties.Settings.Default.allowDuplicates = allowDuplicates.Checked; Properties.Settings.Default.Save(); } + private void UseBundleContainerNameToolStripMenuItem_CheckedChanged(object sender, EventArgs e) { Properties.Settings.Default.useBundleContainerName = useBundleContainerNameToolStripMenuItem.Checked; Properties.Settings.Default.Save(); } + private void skipContainer_CheckedChanged(object sender, EventArgs e) { Properties.Settings.Default.skipContainer = skipContainer.Checked; @@ -2310,6 +2291,7 @@ private void skipContainer_CheckedChanged(object sender, EventArgs e) SkipContainer = skipContainer.Checked; } + private void assetMapTypeMenuItem_DropDownItemClicked(object sender, ToolStripItemClickedEventArgs e) { var assetMapType = Properties.Settings.Default.assetMapType; @@ -2329,6 +2311,7 @@ private void assetMapTypeMenuItem_DropDownItemClicked(object sender, ToolStripIt } } + private void modelsOnly_CheckedChanged(object sender, EventArgs e) { Properties.Settings.Default.modelsOnly = modelsOnly.Checked; @@ -2339,6 +2322,7 @@ private void modelsOnly_CheckedChanged(object sender, EventArgs e) FilterAssetList(); } } + private void enableModelPreview_CheckedChanged(object sender, EventArgs e) { Properties.Settings.Default.enableModelPreview = enableModelPreview.Checked; @@ -2663,9 +2647,9 @@ private async void buildAssetMapToolStripMenuItem_Click(object sender, EventArgs var saveFileDialog = new SaveFileDialog() { - Filter = "Map file (*.map)|*.map", - DefaultExt = "map", - Title = "Select Output File (format will auto adjust according to what you selected)", + Filter = "Map file (*.map)|*.map|MemoryPack AssetMap File|*.memory", + DefaultExt = "map", + Title = "Select Output File (format will auto adjust according to what you selected)", InitialDirectory = saveDirectory, }; @@ -2711,7 +2695,104 @@ private void loadAssetMapToolStripMenuItem_Click(object sender, EventArgs e) assetBrowser.Show(); } + private void exitToolStripMenuItem_Click(object sender, EventArgs e) { Application.Exit(); } + + private void modelsObjectsExportAll_Click(object sender, EventArgs e) + { + this.exportAllObjectssplitToolStripMenuItem1_Click(sender, e); + } + + private void modelsObjectsExportSelected_Click(object sender, EventArgs e) + { + bool includeAnimationClips = modelsIncludeAnimationClips.Checked; + bool mergeObjects = modelsMerge.Checked; + + switch (includeAnimationClips, mergeObjects) + { + case (false, false): + this.exportSelectedObjectsToolStripMenuItem_Click(sender, e); + break; + case (false, true): + this.exportSelectedObjectsmergeToolStripMenuItem_Click(sender, e); + break; + case (true, false): + this.exportObjectswithAnimationClipMenuItem_Click(sender, e); + break; + case (true, true): + this.exportSelectedObjectsmergeWithAnimationClipToolStripMenuItem_Click(sender, e); + break; + } + } + + private void modelsNodesExportSelected_Click(object sender, EventArgs e) + { + bool includeAnimationClips = modelsIncludeAnimationClips.Checked; + + switch(includeAnimationClips) + { + case true: + this.exportSelectedNodessplitSelectedAnimationClipsToolStripMenuItem_Click(sender, e); + break; + case false: + this.exportSelectedNodessplitToolStripMenuItem_Click(sender, e); + break; + } + } + + private void aboutToolStripMenuItem_Click(object sender, EventArgs e) + { + aboutForm = new AboutForm(); + aboutForm.ShowDialog(); + } + + private void gameSelectToolStripMenuItem_Click(object sender, EventArgs e) + { + gameSelector = new GameSelector(this); + gameSelector.ShowDialog(); + } + + private void editUnityCNKeysToolStripMenuItem_Click(object sender, EventArgs e) + { + unityCNEdit = new UnityCNEdit(); + unityCNEdit.ShowDialog(); + } + + #region TexControl + + private static char[] textureChannelNames = new[] { 'B', 'G', 'R', 'A' }; + private bool[] textureChannels = new[] { true, true, true, true }; + + #endregion + + #region GLControl + + private bool glControlLoaded; + private int mdx, mdy; + private bool lmdown, rmdown; + private int pgmID, pgmColorID, pgmBlackID; + private int attributeVertexPosition; + private int attributeNormalDirection; + private int attributeVertexColor; + private int uniformModelMatrix; + private int uniformViewMatrix; + private int uniformProjMatrix; + private int vao; + private OpenTK.Mathematics.Vector3[] vertexData; + private OpenTK.Mathematics.Vector3[] normalData; + private OpenTK.Mathematics.Vector3[] normal2Data; + private OpenTK.Mathematics.Vector4[] colorData; + private Matrix4 modelMatrixData; + private Matrix4 viewMatrixData; + private Matrix4 projMatrixData; + private int[] indiceData; + private int wireFrameMode; + private int shadeMode; + private int normalMode; + + #endregion + #region FMOD + private void FMODinit() { FMODreset(); @@ -2979,9 +3060,11 @@ private bool ERRCHECK(FMOD.RESULT result) } return false; } + #endregion #region GLControl + private void InitOpenTK() { ChangeGLSize(glControl.Size); @@ -3198,71 +3281,7 @@ private void glControl_MouseUp(object sender, MouseEventArgs e) rmdown = false; } } - #endregion - - private void exitToolStripMenuItem_Click(object sender, EventArgs e) - { - Application.Exit(); - } - - private void modelsObjectsExportAll_Click(object sender, EventArgs e) - { - exportAllObjectssplitToolStripMenuItem1_Click(sender, e); - } - - private void modelsObjectsExportSelected_Click(object sender, EventArgs e) - { - bool includeAnimationClips = modelsIncludeAnimationClips.Checked; - bool mergeObjects = modelsMerge.Checked; - - switch ((includeAnimationClips, mergeObjects)) - { - case (false, false): - exportSelectedObjectsToolStripMenuItem_Click(sender, e); - break; - case (false, true): - exportSelectedObjectsmergeToolStripMenuItem_Click(sender, e); - break; - case (true, false): - exportObjectswithAnimationClipMenuItem_Click(sender, e); - break; - case (true, true): - exportSelectedObjectsmergeWithAnimationClipToolStripMenuItem_Click(sender, e); - break; - } - } - - private void modelsNodesExportSelected_Click(object sender, EventArgs e) - { - bool includeAnimationClips = modelsIncludeAnimationClips.Checked; - - switch (includeAnimationClips) - { - case true: - exportSelectedNodessplitSelectedAnimationClipsToolStripMenuItem_Click(sender, e); - break; - case false: - exportSelectedNodessplitToolStripMenuItem_Click(sender, e); - break; - } - } - - private void aboutToolStripMenuItem_Click(object sender, EventArgs e) - { - aboutForm = new AboutForm(); - aboutForm.ShowDialog(); - } - - private void gameSelectToolStripMenuItem_Click(object sender, EventArgs e) - { - gameSelector = new GameSelector(this); - gameSelector.ShowDialog(); - } - private void editUnityCNKeysToolStripMenuItem_Click(object sender, EventArgs e) - { - unityCNEdit = new UnityCNEdit(); - unityCNEdit.ShowDialog(); - } + #endregion } } diff --git a/AnimeStudio.GUI/Studio.cs b/AnimeStudio.GUI/Studio.cs index 4081a2e0..a5e3de32 100644 --- a/AnimeStudio.GUI/Studio.cs +++ b/AnimeStudio.GUI/Studio.cs @@ -268,35 +268,154 @@ private static int ExtractStreamFile(string extractPath, List fileLi public static void UpdateContainers() { - if (exportableAssets.Count > 0) + if (exportableAssets.Count == 0) { - Logger.Info("Updating Containers..."); - foreach (var asset in exportableAssets) + return; + } + + Logger.Info("Updating Containers..."); + var isZZZ = Game.Type.IsZZZ(); + var sourceFileIds = new Dictionary(); + + foreach (var asset in exportableAssets) + { + if (isZZZ && TryUpdateZZZContainer(asset)) { - if (Game.Type.IsZZZ() && ulong.TryParse(asset.Container, out var hash) && Paths.TryGetValue(hash, out var z3Path)) - { - asset.Container = z3Path; - continue; - } - if (int.TryParse(asset.Container, out var value)) + continue; + } + + TryUpdateIndexedContainer(asset, sourceFileIds); + } + + Logger.Info("Updated !!"); + } + + private static bool TryUpdateZZZContainer(AssetItem asset) + { + if (!ulong.TryParse(asset.Container, out var hash) || !Paths.TryGetValue(hash, out var z3Path)) + { + return false; + } + + asset.Container = z3Path; + return true; + } + + private static void TryUpdateIndexedContainer(AssetItem asset, Dictionary sourceFileIds) + { + if (!int.TryParse(asset.Container, out var value)) + { + return; + } + + var id = GetSourceFileId(asset.SourceFile.originalPath, sourceFileIds); + if (!id.HasValue) + { + return; + } + + var last = unchecked((uint)value); + var path = ResourceIndex.GetContainer(id.Value, last); + if (string.IsNullOrEmpty(path)) + { + return; + } + + asset.Container = path; + if (asset.Type == ClassIDType.MiHoYoBinData) + { + asset.Text = Path.GetFileNameWithoutExtension(path); + } + } + + private static uint? GetSourceFileId(string originalPath, Dictionary sourceFileIds) + { + originalPath ??= string.Empty; + if (sourceFileIds.TryGetValue(originalPath, out var id)) + { + return id; + } + + var name = Path.GetFileNameWithoutExtension(originalPath); + id = uint.TryParse(name, out var parsedId) ? parsedId : null; + sourceFileIds.Add(originalPath, id); + return id; + } + + #region Build asset data + + private sealed class AssetDataBuildContext + { + public int ObjectCount { get; } + public int GameObjectCount { get; } + public Dictionary ObjectAssetItems { get; } + public List<(PPtr PPtr, string Name)> MiHoYoBinDataNames { get; } = new List<(PPtr, string)>(); + public List<(PPtr PPtr, string Container)> Containers { get; } = new List<(PPtr, string)>(); + public HashSet FastAssetFilterKeys { get; } + private readonly Dictionary canExportCache = new Dictionary(); + public string AssetBundleName { get; set; } = ""; + public string ProductName { get; set; } + + public AssetDataBuildContext() + { + foreach (var assetsFile in assetsManager.assetsFileList) + { + ObjectCount += assetsFile.Objects.Count; + foreach (var asset in assetsFile.Objects) { - var last = unchecked((uint)value); - var name = Path.GetFileNameWithoutExtension(asset.SourceFile.originalPath); - if (uint.TryParse(name, out var id)) + if (asset is GameObject) { - var path = ResourceIndex.GetContainer(id, last); - if (!string.IsNullOrEmpty(path)) - { - asset.Container = path; - if (asset.Type == ClassIDType.MiHoYoBinData) - { - asset.Text = Path.GetFileNameWithoutExtension(path); - } - } + GameObjectCount++; } } } - Logger.Info("Updated !!"); + + ObjectAssetItems = new Dictionary(ObjectCount); + FastAssetFilterKeys = assetsManager.FilterData.Items + .Select(x => new AssetFilterKey(x.Name, x.PathID, x.Type)) + .ToHashSet(); + } + + public bool CanExport(ClassIDType type) + { + if (!canExportCache.TryGetValue(type, out var canExport)) + { + canExport = type.CanExport(); + canExportCache.Add(type, canExport); + } + + return canExport; + } + } + + private readonly struct AssetFilterKey : IEquatable + { + private readonly string name; + private readonly long pathID; + private readonly ClassIDType type; + + public AssetFilterKey(string name, long pathID, ClassIDType type) + { + this.name = name; + this.pathID = pathID; + this.type = type; + } + + public bool Equals(AssetFilterKey other) + { + return type == other.type + && pathID == other.pathID + && string.Equals(name, other.name, StringComparison.OrdinalIgnoreCase); + } + + public override bool Equals(object obj) + { + return obj is AssetFilterKey other && Equals(other); + } + + public override int GetHashCode() + { + return HashCode.Combine(StringComparer.OrdinalIgnoreCase.GetHashCode(name ?? string.Empty), pathID, type); } } @@ -304,320 +423,373 @@ public static (string, List) BuildAssetData() { StatusStripUpdate("Building asset list..."); - int i = 0; - string productName = null; - var objectCount = assetsManager.assetsFileList.Sum(x => x.Objects.Count); - var objectAssetItemDic = new Dictionary(objectCount); - var mihoyoBinDataNames = new List<(PPtr, string)>(); - var containers = new List<(PPtr, string)>(); + var context = new AssetDataBuildContext(); Progress.Reset(); - Logger.Info($"Loading {objectCount} objects from {assetsManager.assetsFileList.Count} files."); - var assetBundleName = ""; + Logger.Info($"Loading {context.ObjectCount} objects from {assetsManager.assetsFileList.Count} files."); - var fastAssetItemFilterData = new HashSet(assetsManager.FilterData.Items, new AssetFilterDataItemEqualityComparer()); - foreach (var assetsFile in assetsManager.assetsFileList) + if (!BuildExportableAssetItems(context) || !ApplyAssetPostProcessing(context)) { - foreach (var asset in assetsFile.Objects) + return (string.Empty, Array.Empty().ToList()); + } + + visibleAssets = exportableAssets; + + StatusStripUpdate("Building tree structure..."); + var treeNodeCollection = BuildSceneTree(context); + context.ObjectAssetItems.Clear(); + + if (treeNodeCollection == null) + { + return (string.Empty, Array.Empty().ToList()); + } + + return (context.ProductName, treeNodeCollection); + } + + private static List BuildSceneTree(AssetDataBuildContext context) + { + var treeNodeCollection = new List(); + var treeNodeDictionary = new Dictionary(context.GameObjectCount); + var hasFilterData = context.FastAssetFilterKeys.Count > 0; + int j = 0; + Progress.Reset(); + var files = assetsManager.assetsFileList.GroupBy(x => x.originalPath ?? string.Empty).OrderBy(x => x.Key).ToList(); + foreach (var fileGroup in files) + { + var file = fileGroup.Key; + var fileNode = !string.IsNullOrEmpty(file) ? new TreeNode(Path.GetFileName(file)) : null; //RootNode + + foreach (var assetsFile in fileGroup) { - if (assetsManager.tokenSource.IsCancellationRequested) + var assetsFileNode = new TreeNode(assetsFile.fileName); + + foreach (var obj in assetsFile.Objects) { - Logger.Info("Building asset list has been cancelled !!"); - return (string.Empty, Array.Empty().ToList()); + if (assetsManager.tokenSource.IsCancellationRequested) + { + Logger.Info("Building tree structure been cancelled !!"); + return null; + } + + if (obj is not GameObject) + { + if (hasFilterData && IsMissingFromFilter(obj.Name, obj.m_PathID, obj.type, context.FastAssetFilterKeys)) + { + continue; + } + + continue; + } + + var m_GameObject = (GameObject)obj; + var currentNode = GetOrCreateGameObjectNode(m_GameObject, treeNodeDictionary); + AssignAssetTreeNodes(m_GameObject, currentNode, context); + var parentNode = GetParentTreeNode(m_GameObject, assetsFileNode, treeNodeDictionary); + parentNode.Nodes.Add(currentNode); } - var assetItem = new AssetItem(asset); + // TODO: need to do proper cleaning of list to only include filtered assets when required - if (asset is not AssetBundle && asset is not ResourceManager) + if (assetsFileNode.Nodes.Count > 0) { - if (fastAssetItemFilterData.Count > 0 && !fastAssetItemFilterData.Contains(new AssetFilterDataItem { Source = assetItem.SourceFile.fullName, Name = assetItem.Text, PathID = assetItem.m_PathID, Type = assetItem.Type })) + if (fileNode == null) { - Logger.Verbose($"Skipped {(assetItem.Text.Length > 0 ? assetItem.Text : "an asset")} because filter data was set and it was missing from it"); - continue; + treeNodeCollection.Add(assetsFileNode); + } + else + { + fileNode.Nodes.Add(assetsFileNode); } } - - objectAssetItemDic.Add(asset, assetItem); - assetItem.UniqueID = "#" + i; - var exportable = false; - switch (asset) - { - case Texture2D m_Texture2D: - if (!string.IsNullOrEmpty(m_Texture2D.m_StreamData?.path)) - assetItem.FullSize = asset.byteSize + m_Texture2D.m_StreamData.size; - exportable = ClassIDType.Texture2D.CanExport(); - break; - case AudioClip m_AudioClip: - if (!string.IsNullOrEmpty(m_AudioClip.m_Source)) - assetItem.FullSize = asset.byteSize + m_AudioClip.m_Size; - exportable = ClassIDType.AudioClip.CanExport(); - break; - case VideoClip m_VideoClip: - if (!string.IsNullOrEmpty(m_VideoClip.m_OriginalPath)) - assetItem.FullSize = asset.byteSize + m_VideoClip.m_ExternalResources.m_Size; - exportable = ClassIDType.VideoClip.CanExport(); - break; - case PlayerSettings m_PlayerSettings: - productName = m_PlayerSettings.productName; - exportable = ClassIDType.PlayerSettings.CanExport(); - break; - case AssetBundle m_AssetBundle: - assetBundleName = m_AssetBundle.Name; - if (!SkipContainer) - { - foreach (var m_Container in m_AssetBundle.m_Container) - { - var preloadIndex = m_Container.Value.preloadIndex; - var preloadSize = m_Container.Value.preloadSize; - var preloadEnd = preloadIndex + preloadSize; - - switch (preloadIndex) - { - case int n when n < 0: - Logger.Warning($"preloadIndex {preloadIndex} is out of preloadTable range"); - break; - default: - for (int k = preloadIndex; k < preloadEnd; k++) - { - string containerName = m_Container.Key; - if (int.TryParse(m_Container.Key, out _) && Properties.Settings.Default.useBundleContainerName) - { - containerName = assetBundleName; - } - try - { - containers.Add((m_AssetBundle.m_PreloadTable[k], m_Container.Key)); - } catch - { - Logger.Info($"Failed to add container {m_Container.Key}"); - } - } - break; - } - } - } + } - exportable = ClassIDType.AssetBundle.CanExport(); - break; - case IndexObject m_IndexObject: - foreach(var index in m_IndexObject.AssetMap) - { - mihoyoBinDataNames.Add((index.Value.Object, index.Key)); - } + if (fileNode?.Nodes.Count > 0) + { + treeNodeCollection.Add(fileNode); + } - exportable = ClassIDType.IndexObject.CanExport(); - break; - case ResourceManager m_ResourceManager: - foreach (var m_Container in m_ResourceManager.m_Container) - { - containers.Add((m_Container.Value, m_Container.Key)); - } + Progress.Report(++j, files.Count); + } + treeNodeDictionary.Clear(); - exportable = ClassIDType.ResourceManager.CanExport(); - break; - case Mesh _ when ClassIDType.Mesh.CanExport(): - case TextAsset _ when ClassIDType.TextAsset.CanExport(): - case AnimationClip _ when ClassIDType.AnimationClip.CanExport(): - case Font _ when ClassIDType.Font.CanExport(): - case MovieTexture _ when ClassIDType.MovieTexture.CanExport(): - case Sprite _ when ClassIDType.Sprite.CanExport(): - case Material _ when ClassIDType.Material.CanExport(): - case MiHoYoBinData _ when ClassIDType.MiHoYoBinData.CanExport(): - case NapAssetBundleIndexAsset _ when ClassIDType.NapAssetBundleIndexAsset.CanExport(): - case Shader _ when ClassIDType.Shader.CanExport(): - case Animator _ when ClassIDType.Animator.CanExport(): - case MonoBehaviour _ when ClassIDType.MonoBehaviour.CanExport(): - exportable = true; - break; + return treeNodeCollection; + } + + private static GameObjectTreeNode GetOrCreateGameObjectNode(GameObject gameObject, Dictionary treeNodeDictionary) + { + if (!treeNodeDictionary.TryGetValue(gameObject, out var currentNode)) + { + currentNode = new GameObjectTreeNode(gameObject); + treeNodeDictionary.Add(gameObject, currentNode); + } + + return currentNode; + } + + private static void AssignAssetTreeNodes(GameObject gameObject, GameObjectTreeNode currentNode, AssetDataBuildContext context) + { + foreach (var pptr in gameObject.m_Components) + { + if (!pptr.TryGet(out var m_Component)) + { + continue; + } + + AssignTreeNode(m_Component, currentNode, context); + AssignMeshTreeNode(m_Component, currentNode, context); + } + } + + private static void AssignTreeNode(Object asset, GameObjectTreeNode currentNode, AssetDataBuildContext context) + { + if (context.ObjectAssetItems.TryGetValue(asset, out var assetItem)) + { + assetItem.TreeNode = currentNode; + } + } + + private static void AssignMeshTreeNode(Object component, GameObjectTreeNode currentNode, AssetDataBuildContext context) + { + if (component is MeshFilter m_MeshFilter && m_MeshFilter.m_Mesh.TryGet(out var meshFromFilter)) + { + AssignTreeNode(meshFromFilter, currentNode, context); + } + else if (component is SkinnedMeshRenderer m_SkinnedMeshRenderer && m_SkinnedMeshRenderer.m_Mesh.TryGet(out var meshFromRenderer)) + { + AssignTreeNode(meshFromRenderer, currentNode, context); + } + } + + private static TreeNode GetParentTreeNode(GameObject gameObject, TreeNode assetsFileNode, Dictionary treeNodeDictionary) + { + if (gameObject.m_Transform != null) + { + if (gameObject.m_Transform.m_Father.TryGet(out var m_Father)) + { + if (m_Father.m_GameObject.TryGet(out var parentGameObject)) + { + return GetOrCreateGameObjectNode(parentGameObject, treeNodeDictionary); } - if (assetItem.Text == "") + } + } + + return assetsFileNode; + } + + private static bool BuildExportableAssetItems(AssetDataBuildContext context) + { + int i = 0; + var displayAll = Properties.Settings.Default.displayAll; + var hasFilterData = context.FastAssetFilterKeys.Count > 0; + if (displayAll && exportableAssets.Capacity < context.ObjectCount) + { + exportableAssets.Capacity = context.ObjectCount; + } + + foreach (var assetsFile in assetsManager.assetsFileList) + { + foreach (var asset in assetsFile.Objects) + { + if (assetsManager.tokenSource.IsCancellationRequested) { - assetItem.Text = assetItem.TypeString + assetItem.UniqueID; + Logger.Info("Building asset list has been cancelled !!"); + return false; } - if (Properties.Settings.Default.displayAll || exportable) + + if (hasFilterData && asset is not AssetBundle && asset is not ResourceManager && IsMissingFromFilter(asset.Name, asset.m_PathID, asset.type, context.FastAssetFilterKeys)) { - exportableAssets.Add(assetItem); + continue; } - Progress.Report(++i, objectCount); + + AddAssetItem(asset, context, displayAll, i); + Progress.Report(++i, context.ObjectCount); } } - foreach((var pptr, var name) in mihoyoBinDataNames) + + return true; + } + + private static void AddAssetItem(Object asset, AssetDataBuildContext context, bool displayAll, int index) + { + var assetItem = new AssetItem(asset); + + context.ObjectAssetItems.Add(asset, assetItem); + assetItem.UniqueID = "#" + index; + var exportable = ConfigureAssetItemAndReturnExportable(asset, assetItem, context); + + if (assetItem.Text == "") + { + assetItem.Text = assetItem.TypeString + assetItem.UniqueID; + } + if (displayAll || exportable) + { + exportableAssets.Add(assetItem); + } + } + + private static bool ApplyAssetPostProcessing(AssetDataBuildContext context) + { + foreach ((var pptr, var name) in context.MiHoYoBinDataNames) { if (assetsManager.tokenSource.IsCancellationRequested) { Logger.Info("Processing asset names has been cancelled !!"); - return (string.Empty, Array.Empty().ToList()); + return false; } if (pptr.TryGet(out var obj)) { - var assetItem = objectAssetItemDic[obj]; + var assetItem = context.ObjectAssetItems[obj]; if (int.TryParse(name, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var hash)) { assetItem.Text = name; - assetItem.Container = Properties.Settings.Default.useBundleContainerName ? assetBundleName : hash.ToString(); + assetItem.Container = Properties.Settings.Default.useBundleContainerName ? context.AssetBundleName : hash.ToString(); } else assetItem.Text = $"BinFile #{assetItem.m_PathID}"; } } if (!SkipContainer) { - foreach ((var pptr, var container) in containers) + foreach ((var pptr, var container) in context.Containers) { if (assetsManager.tokenSource.IsCancellationRequested) { Logger.Info("Processing containers been cancelled !!"); - return (string.Empty, Array.Empty().ToList()); + return false; } if (pptr.TryGet(out var obj)) { - if (objectAssetItemDic.ContainsKey(obj)) + if (context.ObjectAssetItems.TryGetValue(obj, out var assetItem)) { - objectAssetItemDic[obj].Container = container; + assetItem.Container = container; } } } - containers.Clear(); + context.Containers.Clear(); if (Game.Type.IsGISubGroup() || Game.Type.IsZZZ()) { UpdateContainers(); } } - foreach (var tmp in exportableAssets) - { - if (assetsManager.tokenSource.IsCancellationRequested) - { - Logger.Info("Processing subitems been cancelled !!"); - return (string.Empty, Array.Empty().ToList()); - } - tmp.SetSubItems(); - } - - visibleAssets = exportableAssets; - - StatusStripUpdate("Building tree structure..."); + return true; + } - var treeNodeCollection = new List(); - var treeNodeDictionary = new Dictionary(); - int j = 0; - Progress.Reset(); - var files = assetsManager.assetsFileList.GroupBy(x => x.originalPath ?? string.Empty).OrderBy(x => x.Key).ToDictionary(x => x.Key, x => x.ToList()); - foreach (var (file, assetsFiles) in files) + private static bool ConfigureAssetItemAndReturnExportable(Object asset, AssetItem assetItem, AssetDataBuildContext context) + { + switch (asset) { - var fileNode = !string.IsNullOrEmpty(file) ? new TreeNode(Path.GetFileName(file)) : null; //RootNode - - foreach (var assetsFile in assetsFiles) - { - var assetsFileNode = new TreeNode(assetsFile.fileName); + case Texture2D m_Texture2D: + if (!string.IsNullOrEmpty(m_Texture2D.m_StreamData?.path)) + assetItem.FullSize = asset.byteSize + m_Texture2D.m_StreamData.size; + return context.CanExport(ClassIDType.Texture2D); + case AudioClip m_AudioClip: + if (!string.IsNullOrEmpty(m_AudioClip.m_Source)) + assetItem.FullSize = asset.byteSize + m_AudioClip.m_Size; + return context.CanExport(ClassIDType.AudioClip); + case VideoClip m_VideoClip: + if (!string.IsNullOrEmpty(m_VideoClip.m_OriginalPath)) + assetItem.FullSize = asset.byteSize + m_VideoClip.m_ExternalResources.m_Size; + return context.CanExport(ClassIDType.VideoClip); + case PlayerSettings m_PlayerSettings: + context.ProductName = m_PlayerSettings.productName; + return context.CanExport(ClassIDType.PlayerSettings); + case AssetBundle m_AssetBundle: + context.AssetBundleName = m_AssetBundle.Name; + AddAssetBundleContainers(m_AssetBundle, context); + return context.CanExport(ClassIDType.AssetBundle); + case IndexObject m_IndexObject: + foreach(var index in m_IndexObject.AssetMap) + { + context.MiHoYoBinDataNames.Add((index.Value.Object, index.Key)); + } - foreach (var obj in assetsFile.Objects) + return context.CanExport(ClassIDType.IndexObject); + case ResourceManager m_ResourceManager: + foreach (var m_Container in m_ResourceManager.m_Container) { - if (assetsManager.tokenSource.IsCancellationRequested) - { - Logger.Info("Building tree structure been cancelled !!"); - return (string.Empty, Array.Empty().ToList()); - } + context.Containers.Add((m_Container.Value, m_Container.Key)); + } - var assetItem = new AssetItem(obj); + return context.CanExport(ClassIDType.ResourceManager); + case Mesh _: + return context.CanExport(ClassIDType.Mesh); + case TextAsset _: + return context.CanExport(ClassIDType.TextAsset); + case AnimationClip _: + return context.CanExport(ClassIDType.AnimationClip); + case Font _: + return context.CanExport(ClassIDType.Font); + case MovieTexture _: + return context.CanExport(ClassIDType.MovieTexture); + case Sprite _: + return context.CanExport(ClassIDType.Sprite); + case Material _: + return context.CanExport(ClassIDType.Material); + case MiHoYoBinData _: + return context.CanExport(ClassIDType.MiHoYoBinData); + case NapAssetBundleIndexAsset _: + return context.CanExport(ClassIDType.NapAssetBundleIndexAsset); + case Shader _: + return context.CanExport(ClassIDType.Shader); + case Animator _: + return context.CanExport(ClassIDType.Animator); + case MonoBehaviour _: + return context.CanExport(ClassIDType.MonoBehaviour); + default: + return false; + } + } - if (obj is not GameObject) - { - if (fastAssetItemFilterData.Count > 0 && !fastAssetItemFilterData.Contains(new AssetFilterDataItem { Source = assetItem.SourceFile.fullName, Name = assetItem.Text, PathID = assetItem.m_PathID, Type = assetItem.Type })) - { - Logger.Verbose($"Skipped {(assetItem.Text.Length > 0 ? assetItem.Text : "an asset")} because filter data was set and it was missing from it"); - continue; - } - } + private static void AddAssetBundleContainers(AssetBundle assetBundle, AssetDataBuildContext context) + { + if (SkipContainer) + { + return; + } - if (obj is GameObject m_GameObject) - { - if (!treeNodeDictionary.TryGetValue(m_GameObject, out var currentNode)) - { - currentNode = new GameObjectTreeNode(m_GameObject); - treeNodeDictionary.Add(m_GameObject, currentNode); - } + foreach (var m_Container in assetBundle.m_Container) + { + var preloadIndex = m_Container.Value.preloadIndex; + var preloadSize = m_Container.Value.preloadSize; + var preloadEnd = preloadIndex + preloadSize; - foreach (var pptr in m_GameObject.m_Components) + switch (preloadIndex) + { + case int n when n < 0: + Logger.Warning($"preloadIndex {preloadIndex} is out of preloadTable range"); + break; + default: + for (int k = preloadIndex; k < preloadEnd; k++) + { + try { - if (pptr.TryGet(out var m_Component)) - { - if (objectAssetItemDic.ContainsKey(m_Component)) - { - objectAssetItemDic[m_Component].TreeNode = currentNode; - } - - if (m_Component is MeshFilter m_MeshFilter) - { - if (m_MeshFilter.m_Mesh.TryGet(out var m_Mesh)) - { - if (objectAssetItemDic.ContainsKey(m_Mesh)) - { - objectAssetItemDic[m_Mesh].TreeNode = currentNode; - } - } - } - else if (m_Component is SkinnedMeshRenderer m_SkinnedMeshRenderer) - { - if (m_SkinnedMeshRenderer.m_Mesh.TryGet(out var m_Mesh)) - { - if (objectAssetItemDic.ContainsKey(m_Mesh)) - { - objectAssetItemDic[m_Mesh].TreeNode = currentNode; - } - } - } - } - } - - var parentNode = assetsFileNode; - - if (m_GameObject.m_Transform != null) + context.Containers.Add((assetBundle.m_PreloadTable[k], m_Container.Key)); + } catch { - if (m_GameObject.m_Transform.m_Father.TryGet(out var m_Father)) - { - if (m_Father.m_GameObject.TryGet(out var parentGameObject)) - { - if (!treeNodeDictionary.TryGetValue(parentGameObject, out var parentGameObjectNode)) - { - parentGameObjectNode = new GameObjectTreeNode(parentGameObject); - treeNodeDictionary.Add(parentGameObject, parentGameObjectNode); - } - parentNode = parentGameObjectNode; - } - } + Logger.Info($"Failed to add container {m_Container.Key}"); } - - parentNode.Nodes.Add(currentNode); - } - } - - // TODO: need to do proper cleaning of list to only include filtered assets when required - - if (assetsFileNode.Nodes.Count > 0) - { - if (fileNode == null) - { - treeNodeCollection.Add(assetsFileNode); } - else - { - fileNode.Nodes.Add(assetsFileNode); - } - } - } - - if (fileNode?.Nodes.Count > 0) - { - treeNodeCollection.Add(fileNode); + break; } - - Progress.Report(++j, files.Count); } - treeNodeDictionary.Clear(); + } - objectAssetItemDic.Clear(); + private static bool IsMissingFromFilter(string name, long pathID, ClassIDType type, HashSet fastAssetFilterKeys) + { + if (fastAssetFilterKeys.Count == 0) + { + return false; + } + if (fastAssetFilterKeys.Contains(new AssetFilterKey(name, pathID, type))) + { + return false; + } - return (productName, treeNodeCollection); + Logger.Verbose($"Skipped {(name.Length > 0 ? name : "an asset")} because filter data was set and it was missing from it"); + return true; } + #endregion + public static Dictionary> BuildClassStructure() { var typeMap = new Dictionary>(); @@ -659,6 +831,8 @@ public static Dictionary> BuildClass return typeMap; } + #region Export assets + public static Task ExportAssets(string savePath, List toExportAssets, ExportType exportType, bool openAfterExport) { return Task.Run(() => @@ -669,94 +843,112 @@ public static Task ExportAssets(string savePath, List toExportAssets, int exportedCount = 0; int i = 0; Progress.Reset(); + var assetGroupOption = (AssetGroupOption)Properties.Settings.Default.assetGroupOption; foreach (var asset in toExportAssets) { - string exportPath; - switch ((AssetGroupOption)Properties.Settings.Default.assetGroupOption) - { - case AssetGroupOption.ByType: //type name - exportPath = Path.Combine(savePath, asset.TypeString); - break; - case AssetGroupOption.ByContainer: //container path - if (!string.IsNullOrEmpty(asset.Container)) - { - exportPath = Path.HasExtension(asset.Container) ? Path.Combine(savePath, Path.GetDirectoryName(asset.Container)) : Path.Combine(savePath, asset.Container); - } - else - { - exportPath = savePath; - } - break; - case AssetGroupOption.BySource: //source file - if (string.IsNullOrEmpty(asset.SourceFile.originalPath)) - { - exportPath = Path.Combine(savePath, asset.SourceFile.fileName + "_export"); - } - else - { - exportPath = Path.Combine(savePath, Path.GetFileName(asset.SourceFile.originalPath) + "_export", asset.SourceFile.fileName); - } - break; - default: - exportPath = savePath; - break; - } - exportPath += Path.DirectorySeparatorChar; + var exportPath = GetAssetExportPath(savePath, asset, assetGroupOption); StatusStripUpdate($"[{exportedCount}/{toExportCount}] Exporting {asset.TypeString}: {asset.Text}"); - try - { - switch (exportType) - { - case ExportType.Raw: - if (ExportRawFile(asset, exportPath)) - { - exportedCount++; - } - break; - case ExportType.Dump: - if (ExportDumpFile(asset, exportPath)) - { - exportedCount++; - } - break; - case ExportType.Convert: - if (ExportConvertFile(asset, exportPath)) - { - exportedCount++; - } - break; - case ExportType.JSON: - if (ExportJSONFile(asset, exportPath)) - { - exportedCount++; - } - break; - } - } - catch (Exception ex) + + if (TryExportAsset(asset, exportPath, exportType)) { - Logger.Error($"Export {asset.Type}:{asset.Text} error\r\n{ex.Message}\r\n{ex.StackTrace}"); + exportedCount++; } Progress.Report(++i, toExportCount); } - var statusText = exportedCount == 0 ? "Nothing exported." : $"Finished exporting {exportedCount} assets."; + StatusStripUpdate(GetExportAssetsStatus(toExportCount, exportedCount)); - if (toExportCount > exportedCount) + if (openAfterExport && exportedCount > 0) { - statusText += $" {toExportCount - exportedCount} assets skipped (not extractable or files already exist)"; + OpenFolderInExplorer(savePath); } + }); + } - StatusStripUpdate(statusText); + private static string GetAssetExportPath(string savePath, AssetItem asset, AssetGroupOption assetGroupOption) + { + string exportPath; + switch (assetGroupOption) + { + case AssetGroupOption.ByType: + exportPath = Path.Combine(savePath, asset.TypeString); + break; + case AssetGroupOption.ByContainer: + exportPath = GetContainerExportPath(savePath, asset); + break; + case AssetGroupOption.BySource: + exportPath = GetSourceExportPath(savePath, asset); + break; + default: + exportPath = savePath; + break; + } - if (openAfterExport && exportedCount > 0) + return exportPath + Path.DirectorySeparatorChar; + } + + private static string GetContainerExportPath(string savePath, AssetItem asset) + { + if (string.IsNullOrEmpty(asset.Container)) + { + return savePath; + } + + return Path.HasExtension(asset.Container) + ? Path.Combine(savePath, Path.GetDirectoryName(asset.Container)) + : Path.Combine(savePath, asset.Container); + } + + private static string GetSourceExportPath(string savePath, AssetItem asset) + { + if (string.IsNullOrEmpty(asset.SourceFile.originalPath)) + { + return Path.Combine(savePath, asset.SourceFile.fileName + "_export"); + } + + return Path.Combine(savePath, Path.GetFileName(asset.SourceFile.originalPath) + "_export", asset.SourceFile.fileName); + } + + private static bool TryExportAsset(AssetItem asset, string exportPath, ExportType exportType) + { + try + { + switch (exportType) { - OpenFolderInExplorer(savePath); + case ExportType.Raw: + return ExportRawFile(asset, exportPath); + case ExportType.Dump: + return ExportDumpFile(asset, exportPath); + case ExportType.Convert: + return ExportConvertFile(asset, exportPath); + case ExportType.JSON: + return ExportJSONFile(asset, exportPath); + default: + return false; } - }); + } + catch (Exception ex) + { + Logger.Error($"Export {asset.Type}:{asset.Text} error\r\n{ex.Message}\r\n{ex.StackTrace}"); + return false; + } } + private static string GetExportAssetsStatus(int toExportCount, int exportedCount) + { + var statusText = exportedCount == 0 ? "Nothing exported." : $"Finished exporting {exportedCount} assets."; + + if (toExportCount > exportedCount) + { + statusText += $" {toExportCount - exportedCount} assets skipped (not extractable or files already exist)"; + } + + return statusText; + } + + #endregion + public static Task ExportAssetsList(string savePath, List toExportAssets, ExportListType exportListType) { return Task.Run(() => @@ -807,89 +999,97 @@ public static Task ExportAssetsList(string savePath, List toExportAss }); } + #region Export split objects + public static Task ExportSplitObjects(string savePath, TreeNodeCollection nodes) { return Task.Run(() => { - var exportNodes = GetNodes(nodes); - var count = exportNodes.Cast().Sum(x => x.Nodes.Count); + var exportNodes = GetSplitExportNodes(nodes).ToList(); + var count = exportNodes.Sum(x => x.Nodes.Count); int k = 0; Progress.Reset(); - foreach (TreeNode node in exportNodes) + + foreach (var node in exportNodes) { - //遍历一级子节点 - foreach (GameObjectTreeNode j in node.Nodes) + foreach (GameObjectTreeNode childNode in node.Nodes) { - //收集所有子节点 - var gameObjects = new List(); - CollectNode(j, gameObjects); - //跳过一些不需要导出的object - if (gameObjects.All(x => x.m_SkinnedMeshRenderer == null && x.m_MeshFilter == null)) - { - Progress.Report(++k, count); - continue; - } - //处理非法文件名 - var filename = FixFileName(j.Text); - if (node.Parent != null) - { - filename = Path.Combine(FixFileName(node.Parent.Text), filename); - } - //每个文件存放在单独的文件夹 - var targetPath = $"{savePath}{filename}{Path.DirectorySeparatorChar}"; - //重名文件处理 - for (int i = 1; ; i++) - { - if (Directory.Exists(targetPath)) - { - targetPath = $"{savePath}{filename} ({i}){Path.DirectorySeparatorChar}"; - } - else - { - break; - } - } - Directory.CreateDirectory(targetPath); - //导出FBX - StatusStripUpdate($"Exporting {filename}.fbx"); - try - { - ExportGameObject(j.gameObject, targetPath); - } - catch (Exception ex) - { - Logger.Error($"Export GameObject:{j.Text} error\r\n{ex.Message}\r\n{ex.StackTrace}"); - } - + ExportSplitObject(savePath, node, childNode); Progress.Report(++k, count); - StatusStripUpdate($"Finished exporting {filename}.fbx"); } } + if (Properties.Settings.Default.openAfterExport) { OpenFolderInExplorer(savePath); } StatusStripUpdate("Finished"); + }); + } - IEnumerable GetNodes(TreeNodeCollection nodes) + private static IEnumerable GetSplitExportNodes(TreeNodeCollection nodes) + { + foreach (TreeNode node in nodes) + { + if (node.Nodes.Count == 0) { - foreach(TreeNode node in nodes) - { - var subNodes = node.Nodes.OfType().ToArray(); - if (subNodes.Length == 0) - { - yield return node; - } - else - { - foreach (TreeNode subNode in subNodes) - { - yield return subNode; - } - } - } + yield return node; + continue; } - }); + + foreach (TreeNode subNode in node.Nodes) + { + yield return subNode; + } + } + } + + private static void ExportSplitObject(string savePath, TreeNode parentNode, GameObjectTreeNode node) + { + var gameObjects = new List(); + CollectNode(node, gameObjects); + if (gameObjects.All(x => x.m_SkinnedMeshRenderer == null && x.m_MeshFilter == null)) + { + return; + } + + var filename = GetSplitObjectFileName(parentNode, node); + var targetPath = GetUniqueSplitObjectTargetPath(savePath, filename); + Directory.CreateDirectory(targetPath); + + StatusStripUpdate($"Exporting {filename}.fbx"); + try + { + ExportGameObject(node.gameObject, targetPath); + } + catch (Exception ex) + { + Logger.Error($"Export GameObject:{node.Text} error\r\n{ex.Message}\r\n{ex.StackTrace}"); + } + + StatusStripUpdate($"Finished exporting {filename}.fbx"); + } + + private static string GetSplitObjectFileName(TreeNode parentNode, GameObjectTreeNode node) + { + var filename = FixFileName(node.Text); + if (parentNode.Parent != null) + { + filename = Path.Combine(FixFileName(parentNode.Parent.Text), filename); + } + + return filename; + } + + private static string GetUniqueSplitObjectTargetPath(string savePath, string filename) + { + var targetPath = $"{savePath}{filename}{Path.DirectorySeparatorChar}"; + for (int i = 1; Directory.Exists(targetPath); i++) + { + targetPath = $"{savePath}{filename} ({i}){Path.DirectorySeparatorChar}"; + } + + return targetPath; } private static void CollectNode(GameObjectTreeNode node, List gameObjects) @@ -901,6 +1101,8 @@ private static void CollectNode(GameObjectTreeNode node, List gameOb } } + #endregion + public static Task ExportAnimatorWithAnimationClip(AssetItem animator, List animationList, string exportPath) { return Task.Run(() => diff --git a/AnimeStudio.Utility/AnimeStudio.Utility.csproj b/AnimeStudio.Utility/AnimeStudio.Utility.csproj index 9cb6d201..67fa28a5 100644 --- a/AnimeStudio.Utility/AnimeStudio.Utility.csproj +++ b/AnimeStudio.Utility/AnimeStudio.Utility.csproj @@ -11,9 +11,9 @@ - + - + diff --git a/AnimeStudio/AnimeStudio.csproj b/AnimeStudio/AnimeStudio.csproj index 9db8812b..caabf245 100644 --- a/AnimeStudio/AnimeStudio.csproj +++ b/AnimeStudio/AnimeStudio.csproj @@ -8,12 +8,15 @@ - - - + + + + + - - + + + diff --git a/AnimeStudio/AssetMap.cs b/AnimeStudio/AssetMap.cs index e05517fa..6e27d22e 100644 --- a/AnimeStudio/AssetMap.cs +++ b/AnimeStudio/AssetMap.cs @@ -1,14 +1,15 @@ -using MessagePack; -using System; +using System; using System.Collections.Generic; -using System.Linq; using System.Text.RegularExpressions; +using MemoryPack; +using MessagePack; namespace AnimeStudio { public static class StringCache { private static readonly HashSet _cache = new(StringComparer.Ordinal); + public static string Get(string value) { if (value == null) return null; @@ -19,67 +20,121 @@ public static string Get(string value) _cache.Add(value); return value; } + + public static void Clear() + { + _cache.Clear(); + } } - [MessagePackObject] - public record AssetMap + [MessagePackObject, MemoryPackable] + public partial record AssetMap { [Key(0)] public GameType GameType { get; set; } + [Key(1)] public List AssetEntries { get; set; } } - [MessagePackObject] - public record AssetEntry + + [MessagePackObject, MemoryPackable] + public partial record AssetEntry { - private string _name; private string _container; - private string _source; private string _hash; + private string _name; + private string _source; [Key(0)] public string Name { get => _name; set => _name = StringCache.Get(value); } + [Key(1)] public string Container { get => _container; set => _container = StringCache.Get(value); } + [Key(2)] public string Source { get => _source; set => _source = StringCache.Get(value); } + [Key(3)] public long PathID { get; set; } + [Key(4)] public ClassIDType Type { get; set; } + [Key(5)] public string Hash { get => _hash; - set => _hash = StringCache.Get(value); + set => _hash = value; } + [Key(6)] public long Offset { get; set; } = -1; public bool Matches(Dictionary filters) { - var matches = new List(); - foreach(var filter in filters) + if(filters is null || filters.Count == 0) + return true; + + foreach (KeyValuePair kvp in filters) + { + var regex = kvp.Value; + if (string.IsNullOrEmpty(regex.ToString())) + continue; + + if(!TryGetFilterValue(kvp.Key, out var value)) + return false; + + if(!regex.IsMatch(value)) + return false; + } + + return true; + } + + private bool TryGetFilterValue(string key, out string value) + { + if (string.Equals(key, nameof(Name), StringComparison.OrdinalIgnoreCase)) + { + value = Name; + return true; + } + if (string.Equals(key, nameof(Container), StringComparison.OrdinalIgnoreCase)) { - matches.Add(filter.Key switch - { - string value when value.Equals(nameof(Name), StringComparison.OrdinalIgnoreCase) => filter.Value.IsMatch(Name), - string value when value.Equals(nameof(Container), StringComparison.OrdinalIgnoreCase) => filter.Value.IsMatch(Container), - string value when value.Equals(nameof(Source), StringComparison.OrdinalIgnoreCase) => filter.Value.IsMatch(Source), - string value when value.Equals(nameof(PathID), StringComparison.OrdinalIgnoreCase) => filter.Value.IsMatch(PathID.ToString()), - string value when value.Equals(nameof (Type), StringComparison.OrdinalIgnoreCase) => filter.Value.IsMatch(Type.ToString()), - string value when value.Equals(nameof(Hash), StringComparison.OrdinalIgnoreCase) || value.Equals("SHA256Hash", StringComparison.OrdinalIgnoreCase) => filter.Value.IsMatch(Hash ?? String.Empty) - }); + value = Container; + return true; } - return matches.Count(x => x == true) == filters.Count; + if (string.Equals(key, nameof(Source), StringComparison.OrdinalIgnoreCase)) + { + value = Source; + return true; + } + if (string.Equals(key, nameof(PathID), StringComparison.OrdinalIgnoreCase)) + { + value = PathID.ToString(); + return true; + } + if (string.Equals(key, nameof(Type), StringComparison.OrdinalIgnoreCase)) + { + value = Type.ToString(); + return true; + } + if (string.Equals(key, nameof(Hash), StringComparison.OrdinalIgnoreCase) || + string.Equals(key, "SHA256Hash", StringComparison.OrdinalIgnoreCase)) + { + value = Hash ?? string.Empty; + return true; + } + + value = null; + return false; } } } diff --git a/AnimeStudio/AssetsHelper.cs b/AnimeStudio/AssetsHelper.cs index 4a1d030e..28b9c643 100644 --- a/AnimeStudio/AssetsHelper.cs +++ b/AnimeStudio/AssetsHelper.cs @@ -1,16 +1,19 @@ using System; -using System.IO; -using System.Linq; using System.Collections.Generic; -using System.Threading; using System.Globalization; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json; +using System.IO; +using System.Linq; +using System.Text; using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; using System.Xml; -using System.Text; +using MemoryPack; +using MemoryPack.Streaming; using MessagePack; -using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Formatting = Newtonsoft.Json.Formatting; namespace AnimeStudio { @@ -18,7 +21,7 @@ public static class AssetsHelper { public const string MapName = "Maps"; - public static bool Minimal = true; + public static bool Minimal = true; public static CancellationTokenSource tokenSource = new CancellationTokenSource(); private static string BaseFolder = ""; @@ -28,13 +31,6 @@ public static class AssetsHelper public static Dictionary Paths { get; set; } = new Dictionary(); - public record Entry - { - public string Path { get; set; } - public long Offset { get; set; } - public List Dependencies { get; set; } - } - public static void SetUnityVersion(string version) { assetsManager.SpecifyUnityVersion = version; @@ -317,7 +313,7 @@ private static void ParseCABMap(BinaryReader reader) }; CABMap.Add(cab, entry); } - } + } public static async Task BuildAssetMap(string[] files, string mapName, Game game, string savePath, ExportListType exportListType, ClassIDType[] typeFilters = null, Regex[] nameFilters = null, Regex[] containerFilters = null) { @@ -527,7 +523,12 @@ private static void BuildAssetMap(string file, List assets, ClassIDT })); } - public static string[] ParseAssetMap(string mapName, ExportListType mapType, ClassIDType[] typeFilter, Regex[] nameFilter, Regex[] containerFilter) + public static string[] ParseAssetMap + (string mapName, + ExportListType mapType, + ClassIDType[] typeFilter, + Regex[] nameFilter, + Regex[] containerFilter) { var matches = new HashSet(); @@ -595,7 +596,7 @@ public static string[] ParseAssetMap(string mapName, ExportListType mapType, Cla using var file = new StreamReader(stream); using var reader = new JsonTextReader(file); - var serializer = new JsonSerializer() { Formatting = Newtonsoft.Json.Formatting.Indented }; + var serializer = new JsonSerializer { Formatting = Formatting.Indented }; serializer.Converters.Add(new StringEnumConverter()); var entries = serializer.Deserialize>(reader); @@ -610,10 +611,32 @@ public static string[] ParseAssetMap(string mapName, ExportListType mapType, Cla } } } + + break; + case ExportListType.MemoryPack: + { + using FileStream stream = File.OpenRead(mapName); + AssetMap assetMap = MemoryPackStreamingSerializer.DeserializeAsync + (stream).FirstAsync().GetAwaiter().GetResult(); + foreach (AssetEntry entry in assetMap.AssetEntries) + { + if(entry == null) continue; + + bool isNameMatch = nameFilter.Length == 0 || nameFilter.Any + (x => x.IsMatch(entry.Name ?? string.Empty)); + bool isContainerMatch = containerFilter.Length == 0 || containerFilter.Any + (x => x.IsMatch(entry.Container ?? string.Empty)); + bool isTypeMatch = typeFilter.Length == 0 || typeFilter.Any(x => x == entry.Type); + + if(isNameMatch && isContainerMatch && isTypeMatch) + matches.Add(entry.Source ?? string.Empty); + } + + } break; } - + return matches.ToArray(); } @@ -648,72 +671,90 @@ private static void UpdateContainers(List assets, Game game) private static Task ExportAssetsMap(List toExportAssets, Game game, string name, string savePath, ExportListType exportListType) { - return Task.Run(() => - { - Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US"); - - Progress.Reset(); - - string filename = string.Empty; - if (exportListType.Equals(ExportListType.None)) - { - Logger.Info($"No export list type has been selected, skipping..."); - } - else - { - if (exportListType.HasFlag(ExportListType.XML)) - { - filename = Path.Combine(savePath, $"{name}.xml"); - var xmlSettings = new XmlWriterSettings() { Indent = true }; - using XmlWriter writer = XmlWriter.Create(filename, xmlSettings); - writer.WriteStartDocument(); - writer.WriteStartElement("Assets"); - writer.WriteAttributeString("filename", filename); - writer.WriteAttributeString("createdAt", DateTime.UtcNow.ToString("s")); - foreach (var asset in toExportAssets) - { - writer.WriteStartElement("Asset"); - writer.WriteElementString("Name", asset.Name); - writer.WriteElementString("Container", asset.Container); - writer.WriteStartElement("Type"); - writer.WriteAttributeString("id", ((int)asset.Type).ToString()); - writer.WriteValue(asset.Type.ToString()); - writer.WriteEndElement(); - writer.WriteElementString("PathID", asset.PathID.ToString()); - writer.WriteElementString("Source", asset.Source); - writer.WriteEndElement(); - } - writer.WriteEndElement(); - writer.WriteEndDocument(); - } - if (exportListType.HasFlag(ExportListType.JSON)) - { - filename = Path.Combine(savePath, $"{name}.json"); - using StreamWriter file = File.CreateText(filename); - var serializer = new JsonSerializer() { Formatting = Newtonsoft.Json.Formatting.Indented }; - serializer.Converters.Add(new StringEnumConverter()); - serializer.Serialize(file, new - { - GameType = game.Type, - AssetEntries = toExportAssets - }); - } - if (exportListType.HasFlag(ExportListType.MessagePack)) - { - filename = Path.Combine(savePath, $"{name}.map"); - using var file = File.Create(filename); - var assetMap = new AssetMap - { - GameType = game.Type, - AssetEntries = toExportAssets - }; - MessagePackSerializer.Serialize(file, assetMap, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray)); - } - - Logger.Info($"Finished buidling AssetMap with {toExportAssets.Count} assets."); - } - }); + return Task.Run + (async () => + { + Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US"); + + Progress.Reset(); + + string filename = string.Empty; + if (exportListType.Equals(ExportListType.None)) + { + Logger.Info($"No export list type has been selected, skipping..."); + } + else + { + if (exportListType.HasFlag(ExportListType.XML)) + { + filename = Path.Combine(savePath, $"{name}.xml"); + var xmlSettings = new XmlWriterSettings() { Indent = true }; + using XmlWriter writer = XmlWriter.Create(filename, xmlSettings); + writer.WriteStartDocument(); + writer.WriteStartElement("Assets"); + writer.WriteAttributeString("filename", filename); + writer.WriteAttributeString("createdAt", DateTime.UtcNow.ToString("s")); + foreach (var asset in toExportAssets) + { + writer.WriteStartElement("Asset"); + writer.WriteElementString("Name", asset.Name); + writer.WriteElementString("Container", asset.Container); + writer.WriteStartElement("Type"); + writer.WriteAttributeString("id", ((int)asset.Type).ToString()); + writer.WriteValue(asset.Type.ToString()); + writer.WriteEndElement(); + writer.WriteElementString("PathID", asset.PathID.ToString()); + writer.WriteElementString("Source", asset.Source); + writer.WriteEndElement(); + } + writer.WriteEndElement(); + writer.WriteEndDocument(); + } + if (exportListType.HasFlag(ExportListType.JSON)) + { + filename = Path.Combine(savePath, $"{name}.json"); + using StreamWriter file = File.CreateText(filename); + var serializer = new JsonSerializer { Formatting = Formatting.Indented }; + serializer.Converters.Add(new StringEnumConverter()); + serializer.Serialize(file, new + { + GameType = game.Type, + AssetEntries = toExportAssets + }); + } + if (exportListType.HasFlag(ExportListType.MessagePack)) + { + filename = Path.Combine(savePath, $"{name}.map"); + using var file = File.Create(filename); + var assetMap = new AssetMap + { + GameType = game.Type, + AssetEntries = toExportAssets + }; + MessagePackSerializer.Serialize(file, assetMap, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray)); + } + + if(exportListType.HasFlag(ExportListType.MemoryPack)) + { + filename = Path.Combine(savePath, $"{name}.memory"); + var assetMap = new AssetMap + { + GameType = game.Type, + AssetEntries = toExportAssets + }; + + var assetMaps = new List(); + assetMaps.Add(assetMap); + + byte[] data = MemoryPackSerializer.Serialize(assetMaps); + File.WriteAllBytes(filename, data); + } + + Logger.Info($"Finished buidling AssetMap with {toExportAssets.Count} assets."); + } + }); } + public static async Task BuildBoth(string[] files, string mapName, string baseFolder, Game game, string savePath, ExportListType exportListType, ClassIDType[] typeFilters = null, Regex[] nameFilters = null, Regex[] containerFilters = null) { Logger.Info($"Building Both..."); @@ -735,5 +776,16 @@ public static async Task BuildBoth(string[] files, string mapName, string baseFo Logger.Info($"Map build successfully !! {collision} collisions found"); await ExportAssetsMap(assets, game, mapName, savePath, exportListType); } + + #region Nested type: Entry + + public record Entry + { + public string Path { get; set; } + public long Offset { get; set; } + public List Dependencies { get; set; } + } + + #endregion } } diff --git a/AnimeStudio/Classes/Object.cs b/AnimeStudio/Classes/Object.cs index 5cabb59d..b50bbfbb 100644 --- a/AnimeStudio/Classes/Object.cs +++ b/AnimeStudio/Classes/Object.cs @@ -1,7 +1,6 @@ -using K4os.Hash.xxHash; +using System.Collections.Specialized; +using K4os.Hash.xxHash; using Newtonsoft.Json; -using System; -using System.Collections.Specialized; namespace AnimeStudio { @@ -9,24 +8,25 @@ public class Object { [JsonIgnore] public SerializedFile assetsFile; - [JsonIgnore] - public ObjectReader reader; - [JsonIgnore] - public long m_PathID; - [JsonIgnore] - public int[] version; + [JsonIgnore] protected BuildType buildType; + + [JsonIgnore] public uint byteSize; + + [JsonIgnore] public long m_PathID; + [JsonIgnore] public BuildTarget platform; - [JsonIgnore] - public ClassIDType type; + + [JsonIgnore] public ObjectReader reader; + [JsonIgnore] public SerializedType serializedType; - [JsonIgnore] - public uint byteSize; - public virtual string Name => string.Empty; + [JsonIgnore] public ClassIDType type; + + [JsonIgnore] public int[] version; public Object(ObjectReader reader) { @@ -49,6 +49,11 @@ public Object(ObjectReader reader) } } + public virtual string Name + { + get => string.Empty; + } + public string GetHash(bool overrides = false) { if (type == ClassIDType.Texture2D && overrides) @@ -105,7 +110,7 @@ public byte[] GetRawData() Logger.Verbose($"Dumping raw bytes of the object with {m_PathID} in file {assetsFile.fileName}..."); long pos = reader.Position; reader.Reset(); - byte[] data = reader.ReadBytes((int)byteSize); + byte[] data = reader.ReadBytes((int) byteSize); reader.Position = pos; return data; } diff --git a/AnimeStudio/EndianBinaryReader.cs b/AnimeStudio/EndianBinaryReader.cs index c06942ca..dd6246ed 100644 --- a/AnimeStudio/EndianBinaryReader.cs +++ b/AnimeStudio/EndianBinaryReader.cs @@ -165,19 +165,32 @@ public string ReadAlignedString() public string ReadStringToNull(int maxLength = 32767) { - var bytes = new List(); - int count = 0; - while (Remaining > 0 && count < maxLength) + var length = (int)Math.Min(Remaining, maxLength); + if (length <= 0) { - var b = ReadByte(); - if (b == 0) + return string.Empty; + } + + var bytes = ArrayPool.Shared.Rent(length); + var count = 0; + try + { + while (count < length) { - break; + var b = ReadByte(); + if (b == 0) + { + break; + } + bytes[count++] = b; } - bytes.Add(b); - count++; + + return Encoding.UTF8.GetString(bytes, 0, count); + } + finally + { + ArrayPool.Shared.Return(bytes); } - return Encoding.UTF8.GetString(bytes.ToArray()); } public Quaternion ReadQuaternion() diff --git a/AnimeStudio/ExportTypeList.cs b/AnimeStudio/ExportTypeList.cs index 557c39de..71adfddf 100644 --- a/AnimeStudio/ExportTypeList.cs +++ b/AnimeStudio/ExportTypeList.cs @@ -8,6 +8,7 @@ public enum ExportListType None, MessagePack, XML, - JSON = 4, + JSON = 4, + MemoryPack = 8 } } diff --git a/AnimeStudio/ObjectReader.cs b/AnimeStudio/ObjectReader.cs index 25d39ada..a6415f0a 100644 --- a/AnimeStudio/ObjectReader.cs +++ b/AnimeStudio/ObjectReader.cs @@ -1,22 +1,18 @@ using System; -using System.IO; namespace AnimeStudio { public class ObjectReader : EndianBinaryReader { - public SerializedFile assetsFile; - public Game Game; - public long m_PathID; - public long byteStart; - public uint byteSize; - public ClassIDType type; - public SerializedType serializedType; - public BuildTarget platform; + public SerializedFile assetsFile; + public uint byteSize; + public long byteStart; + public Game Game; + public long m_PathID; public SerializedFileFormatVersion m_Version; - - public int[] version => assetsFile.version; - public BuildType buildType => assetsFile.buildType; + public BuildTarget platform; + public SerializedType serializedType; + public ClassIDType type; public ObjectReader(EndianBinaryReader reader, SerializedFile assetsFile, ObjectInfo objectInfo, Game game) : base(reader.BaseStream, reader.Endian) { @@ -41,13 +37,23 @@ public ObjectReader(EndianBinaryReader reader, SerializedFile assetsFile, Object Logger.Verbose($"Initialized reader for {type} object with {m_PathID} in file {assetsFile.fileName} !!"); } + public int[] version + { + get => assetsFile.version; + } + + public BuildType buildType + { + get => assetsFile.buildType; + } + public override int Read(byte[] buffer, int index, int count) { var pos = Position - byteStart; - if (pos + count > byteSize) - { - throw new EndOfStreamException("Unable to read beyond the end of the stream."); - } + //if (pos + count > byteSize) + //{ + //throw new EndOfStreamException("Unable to read beyond the end of the stream."); + //} return base.Read(buffer, index, count); } diff --git a/AnimeStudio/ResourceMap.cs b/AnimeStudio/ResourceMap.cs index 41211ac7..298f52b0 100644 --- a/AnimeStudio/ResourceMap.cs +++ b/AnimeStudio/ResourceMap.cs @@ -3,6 +3,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using MemoryPack; +using MemoryPack.Streaming; namespace AnimeStudio { @@ -31,27 +34,56 @@ public static int FromFile(string path) Logger.Info(string.Format("Parsing....")); try { - var extension = Path.GetExtension(path).ToLower(); - using var stream = File.OpenRead(path); - - if (extension == ".map") + string extension = Path.GetExtension(path).ToLower(); + using FileStream stream = File.OpenRead(path); + AssetMap newMap = null; + + switch(extension) { - // Deserialize map - Instance = MessagePackSerializer.Deserialize(stream, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray)); + case ".map": + { + // Deserialize map + newMap = MessagePackSerializer.Deserialize + (stream, + MessagePackSerializerOptions.Standard.WithCompression + (MessagePackCompression.Lz4BlockArray)); + break; + } + + case ".json": + { + // Deserialize json + using var reader = new StreamReader(stream); + string jsonContent = reader.ReadToEnd(); + AssetMap parsed = JsonConvert.DeserializeObject(jsonContent); + + newMap = new AssetMap + { + GameType = parsed.GameType, + AssetEntries = parsed.AssetEntries + }; + break; + } + case ".memory": + { + ReadOnlySpan bytes = File.ReadAllBytes(path); + List assetMaps = MemoryPackSerializer.Deserialize> + (bytes); + + AssetMap assetMap = assetMaps.FirstOrDefault(); + newMap = assetMap; + break; + } } - else if (extension == ".json") + + if (newMap == null) { - // Deserialize json - using var reader = new StreamReader(stream); - var jsonContent = reader.ReadToEnd(); - var parsed = JsonConvert.DeserializeObject(jsonContent); + Logger.Error("AssetMap was not loaded"); + return -1; + } - Instance = new AssetMap - { - GameType = parsed.GameType, - AssetEntries = parsed.AssetEntries - }; - } + StringCache.Clear(); + Instance = newMap; } catch (Exception e) { @@ -72,6 +104,7 @@ public static void Clear() { Instance.GameType = GameType.Normal; Instance.AssetEntries = new List(); + StringCache.Clear(); } } }