Resource Imports

Volatility/CLI/Commands/AutotestCommand.cs

@@ -48,7 +48,7 @@ public async Task Execute()
         TexturePC textureHeaderPC = new TexturePC
         {
             AssetName = "autotest_header_PC",
-            ResourceID = GetResourceIDFromName("autotest_header_PC", Endian.LE),
+            ResourceID = GetResourceIDFromName("autotest_header_PC"),
             Format = D3DFORMAT.D3DFMT_DXT1,
             Width = 1024,
             Height = 512,
@@ -62,7 +62,7 @@ public async Task Execute()
         TextureBPR textureHeaderBPR = new TextureBPR
         {
             AssetName = "autotest_header_BPR",
-            ResourceID = GetResourceIDFromName("autotest_header_BPR", Endian.LE),
+            ResourceID = GetResourceIDFromName("autotest_header_BPR"),
             Format = DXGI_FORMAT.DXGI_FORMAT_BC1_UNORM,
             Width = 1024,
             Height = 512,
@@ -77,7 +77,7 @@ public async Task Execute()
 
         textureHeaderBPR.SetResourceArch(Arch.x64);
         textureHeaderBPR.AssetName = "autotest_header_BPRx64";
-        textureHeaderBPR.ResourceID = GetResourceIDFromName(textureHeaderBPR.AssetName, Endian.LE);
+        textureHeaderBPR.ResourceID = GetResourceIDFromName(textureHeaderBPR.AssetName);
 
         // Write 64 bit test BPR header
         TestHeaderRW("autotest_header_BPRx64.dat", textureHeaderBPR);
@@ -86,7 +86,7 @@ public async Task Execute()
         TexturePS3 textureHeaderPS3 = new TexturePS3
         {
             AssetName = "autotest_header_PS3",
-            ResourceID = GetResourceIDFromName("autotest_header_PS3", Endian.BE),
+            ResourceID = GetResourceIDFromName("autotest_header_PS3"),
             Format = CELL_GCM_COLOR_FORMAT.CELL_GCM_TEXTURE_COMPRESSED_DXT45,
             Width = 1024,
             Height = 512,
@@ -100,7 +100,7 @@ public async Task Execute()
         TextureX360 textureHeaderX360 = new TextureX360
         {
             AssetName = "autotest_header_X360",
-            ResourceID = GetResourceIDFromName("autotest_header_X360", Endian.BE),
+            ResourceID = GetResourceIDFromName("autotest_header_X360"),
             Format = new GPUTEXTURE_FETCH_CONSTANT
             {
                 Tiled = true,

Volatility/CLI/Commands/ImportResourceCommand.cs

@@ -72,8 +72,9 @@ public async Task Execute()
 				var serializer = new SerializerBuilder()
     				.DisableAliases()
     				.WithTypeInspector(inner => new IncludeFieldsTypeInspector(inner))
-    				.WithTypeConverter(new ResourceYamlTypeConverter())
-    				.WithTypeConverter(new StringEnumYamlTypeConverter())
+                    .WithTypeConverter(new ResourceYamlTypeConverter())
+                    .WithTypeConverter(new ResourceIDYamlTypeConverter())
+                    .WithTypeConverter(new StringEnumYamlTypeConverter())
     				.Build();
 
 				var serializedString = new string("");

Volatility/Resources/EnvironmentKeyframe/EnvironmentKeyframe.cs

@@ -157,10 +157,13 @@ public VignetteData(ResourceBinaryReader reader)
 
 public struct TintData 
 {
-    public ulong ColorCubePtr; // TODO: Update with new ResourceID system
+    public ResourceImport ColorCubeReference;
     public TintData(ResourceBinaryReader reader, Arch arch) 
     {
-        ColorCubePtr = (arch == Arch.x64 ? reader.ReadUInt64() : reader.ReadUInt32());
+        ColorCubeReference.ReferenceID = (arch == Arch.x64 ? reader.ReadUInt64() : reader.ReadUInt32());
+
+        if (ResourceImport.ReadExternalImport(0, reader, 0x240, out ResourceImport ExternalReference))
+            ColorCubeReference = ExternalReference;
     }
 }
 

Volatility/Resources/InstanceList/InstanceList.cs

@@ -67,10 +67,10 @@ public override void ParseFromStream(ResourceBinaryReader reader, Endian endiann
                 // Padding1 = _padding1, Padding2 = _padding2,
                 MaxVisibleDistanceSquared = _maxVisibleDistanceSquared,
                 Transform = _transform,
-                ResourceId = new ResourceID
+                ResourceId = new ResourceImport
                 {
-                    ID = reader.ReadBytes(4),
-                    Endian = reader.GetEndianness()
+                    ReferenceID = reader.ReadUInt32(),
+                    ExternalImport = false
                 },
             });
         }
@@ -80,7 +80,7 @@ public override void ParseFromStream(ResourceBinaryReader reader, Endian endiann
 public struct Instance
 {
     [EditorLabel("Resource ID"), EditorCategory("InstanceList/Instances"), EditorTooltip("The reference to the resource placed by this instance.")]
-    public ResourceID ResourceId;
+    public ResourceImport ResourceId;
 
     [EditorLabel("Transform"), EditorCategory("InstanceList/Instances"), EditorTooltip("The location, rotation, and scale of this instance.")]
     public Transform Transform;

Volatility/Resources/Model/Model.cs

@@ -60,8 +60,7 @@ public override void WriteToStream(EndianAwareBinaryWriter writer, Endian endian
         // Resource ID References
         for (int i = 0; i < models; i++)
         {
-            writer.Write(ModelDatas[i].ResourceReference.Endian == Endian.BE ? new byte[4] : ModelDatas[i].ResourceReference.ID);
-            writer.Write(ModelDatas[i].ResourceReference.Endian == Endian.LE ? new byte[4] : ModelDatas[i].ResourceReference.ID);
+            writer.Write(ModelDatas[i].ResourceReference.ReferenceID);
             writer.Write(renderablesPtr + (i * 0x4));
             writer.Write((uint)0x0); // Unknown. Always 0 in BPR, not always 0 on X360
         }
@@ -97,9 +96,16 @@ public override void ParseFromStream(ResourceBinaryReader reader, Endian endiann
 
         Flags = reader.ReadByte();
 
+        var maxLength = new[]
+        {
+            lodDistancesPtr + numRenderables * sizeof(uint),
+            renderablesPtr + numRenderables * sizeof(uint),
+            renderableStatesPtr + numRenderables
+        }.Max();
+
         // This currently does a lot of seeking.
         // It may improve performance if we separate this.
-        for (uint i = 0; i < numRenderables; i++)
+        for (int i = 0; i < numRenderables; i++)
         {
             ModelData modelData = new ModelData();
 
@@ -120,8 +126,7 @@ public override void ParseFromStream(ResourceBinaryReader reader, Endian endiann
                 (reader.GetEndianness() == Endian.BE ? 0x4 : 0x0), SeekOrigin.Begin
             );
 
-            modelData.ResourceReference.ID = reader.ReadBytes(4);
-            modelData.ResourceReference.Endian = reader.GetEndianness();
+            ResourceImport.ReadExternalImport(i, reader, maxLength, out modelData.ResourceReference);
 
             ModelDatas.Add(modelData);
         }
@@ -134,7 +139,7 @@ public Model(string path, Endian endianness = Endian.Agnostic) : base(path, endi
     public struct ModelData
     {
         [EditorCategory("Model Data"), EditorLabel("Resource Reference")]
-        public ResourceID ResourceReference;
+        public ResourceImport ResourceReference;
 
         [EditorCategory("Model Data"), EditorLabel("Model State")]
         public State State;

Volatility/Resources/Resource.cs

@@ -5,7 +5,7 @@ namespace Volatility.Resources;
 public abstract class Resource
 {
     [EditorCategory("Resource Info"), EditorLabel("Resource ID"), EditorTooltip("The CRC32 ID used to identify the resource in the game engine.")]
-    public string ResourceID = "";
+    public ResourceID ResourceID = 0x00000000;
 
     [EditorCategory("Resource Info"), EditorLabel("Asset Name"), EditorTooltip("The asset's name in the resource depot.")]
     public string AssetName = "invalid";
@@ -68,30 +68,33 @@ public Resource(string path, Endian endianness = Endian.Agnostic)
             Unpacker = GetUnpackerFromFileName(Path.GetFileName(ImportedFileName));
             if (Unpacker != Unpacker.Raw)
             {
+                int idx = name.LastIndexOf('_');
                 name = Unpacker switch
                 {
                     Unpacker.DGI => name.Replace("_", ""),
-                    Unpacker.Bnd2Manager => name.Substring(0, name.LastIndexOf('_')),
-                    Unpacker.YAP => name.Substring(0, name.LastIndexOf('_'))
+                    Unpacker.Bnd2Manager or Unpacker.YAP when idx > 0 => name[..idx],
+                    Unpacker.Bnd2Manager or Unpacker.YAP => name,                    
+                    _ => name
                 };
             }
             if (ValidateResourceID(name))
             {
                 // We store ResourceIDs how BE platforms do to be consistent with the original console releases.
                 // This makes it easy to cross reference assets between all platforms.
-                ResourceID = (importEndianness == Endian.LE && Unpacker != Unpacker.YAP)
+                ResourceID = Convert.ToUInt64((importEndianness == Endian.LE && Unpacker != Unpacker.YAP)
                     ? FlipResourceIDEndian(name)
-                    : name;
+                    : name
+                    , 16);
 
-                string newName = GetNameByResourceID(ResourceID, GetResourceType().ToString());
+                string newName = GetNameByResourceID(ResourceID);
                 AssetName = !string.IsNullOrEmpty(newName)
                     ? newName
-                    : ResourceID;
+                    : ResourceID.ToString();
             }
             else
             {
                 // TODO: Add new entry to ResourceDB
-                ResourceID = GetResourceIDFromName(name, importEndianness);
+                ResourceID = Convert.ToUInt64(GetResourceIDFromName(name, importEndianness), 16);
                 AssetName = name;
             }
 
@@ -105,23 +108,18 @@ public Resource(string path, Endian endianness = Endian.Agnostic)
 
     private static Unpacker GetUnpackerFromFileName(string filename)
     {
-        if (filename.EndsWith("_1.bin")) // bnd2-manager
+        var name = Path.GetFileName(filename);
+        return name switch
         {
-            return Unpacker.Bnd2Manager;
-        }
-        else if (filename.EndsWith("_primary.dat")) // YAP
-        {
-            return Unpacker.YAP;
-        }
-        else if (filename.EndsWith(".dat")) // DGI
-        {
-            return Unpacker.DGI;
-        }
-        return Unpacker.Raw;
-
-        // Volatility doesn't have a bundle unpacker yet...
+            var n when n.EndsWith("_1.bin", StringComparison.OrdinalIgnoreCase) => Unpacker.Bnd2Manager,
+            var n when n.EndsWith("_primary.dat", StringComparison.OrdinalIgnoreCase) => Unpacker.YAP,
+            var n when n.EndsWith(".dat", StringComparison.OrdinalIgnoreCase)
+                      && n.Count(c => c == '_') == 3 => Unpacker.DGI,
+            var n when n.EndsWith(".dat", StringComparison.OrdinalIgnoreCase) => Unpacker.YAP,
+            _ => Unpacker.Raw,
+        };
     }
-    
+
     public virtual void PushAll() { }
     public virtual void PullAll() { }
 }

Volatility/Resources/ResourceImport.cs

@@ -0,0 +1,90 @@
+using YamlDotNet.Serialization;
+
+using static Volatility.Utilities.ResourceIDUtilities;
+
+namespace Volatility.Resources;
+
+public struct ResourceImport
+{
+    // The idea here is that if the name is populated but
+    // the ID is empty, the name will be calculated into an ID
+    // on export. If both a name and ID exist, use the ID, as
+    // this will keep consistency for imported assets. If you
+    // want to use the calculated name, clear the ReferenceID field.
+    public string Name;
+    public ResourceID ReferenceID;
+    public bool ExternalImport;
+
+    public ResourceImport() 
+    {
+        Name = string.Empty;
+    }
+
+    public ResourceImport(ResourceID id, bool externalImport = false, bool useCalculatedName = false)
+    {
+        ReferenceID = id;
+        ExternalImport = externalImport;
+        Name = GetNameByResourceID(id);
+
+        if (Name.Length > 0 && useCalculatedName)
+        {
+            ReferenceID = 0x0;
+        }
+    }
+
+    public ResourceImport(string name, bool externalImport = false)
+    {
+        Name = name;
+        ExternalImport = externalImport;
+    }
+
+    public static bool ReadExternalImport(int index, EndianAwareBinaryReader reader, long importBlockOffset, out ResourceImport resourceImport)
+    {
+        // In-resource imports block
+        if (reader.BaseStream.Length >= importBlockOffset + (0x10 * index) + 0x10)
+        {
+            long originalPosition = reader.BaseStream.Position;
+
+            reader.BaseStream.Seek(importBlockOffset + (0x10 * index), SeekOrigin.Begin);
+
+            resourceImport = new ResourceImport(reader.ReadUInt64(), externalImport: true);
+
+            reader.BaseStream.Seek(originalPosition, SeekOrigin.Begin);
+
+            return true;
+        }
+        // YAP imports yaml
+        else if (reader.BaseStream is FileStream fs)
+        {
+            string baseName = Path.GetFileNameWithoutExtension(fs.Name);
+
+            string directory = Path.GetDirectoryName(fs.Name);
+
+            resourceImport = new ResourceImport(GetYAMLImportValueAt(Path.Combine(directory, baseName + "_imports.yaml"), index), externalImport: true);
+
+            return true;
+        }
+
+        resourceImport = default;
+        return false;
+    }
+
+
+    public static ResourceID GetYAMLImportValueAt(string yamlPath, int index)
+    {
+        var yaml = File.ReadAllText(yamlPath);
+        var deser = new DeserializerBuilder().Build();
+
+        var list = deser
+            .Deserialize<List<Dictionary<string, string>>>(yaml)
+            ?? throw new InvalidDataException("Expected a YAML sequence of mappings.");
+
+        if (index < 0 || index >= list.Count)
+            throw new ArgumentOutOfRangeException(nameof(index), $"Valid range 0–{list.Count - 1}");
+
+        var kv = list[index].Values.GetEnumerator();
+        kv.MoveNext();
+        return Convert.ToUInt32(kv.Current, 16);
+    }
+};
+

Volatility/Types.cs

@@ -21,25 +21,11 @@ public struct Transform
     public Vector3 Scale;    
 }
 
-// Experimenting with a new way to store ResourceIDs.
-public struct ResourceID
+public readonly struct ResourceID
 {
-    [Newtonsoft.Json.JsonIgnore]
-    public byte[] ID;
-
-    public string HexID
-    {
-        get => BitConverter.ToString(ID).Replace("-", "").ToLower();
-        set => ID = Enumerable.Range(0, value.Length / 2)
-                              .Select(x => Convert.ToByte(value.Substring(x * 2, 2), 16))
-                              .ToArray();
-    }
-
-    public Endian Endian;
-
-    public ResourceID()
-    {
-        ID = new byte[4];
-        Endian = default;
-    }
-}
+    public readonly ulong Value;
+    public ResourceID(ulong v) => Value = v;
+    public static implicit operator ulong(ResourceID r) => r.Value;
+    public static implicit operator ResourceID(ulong v) => new ResourceID(v);
+    public override string ToString() => $"{Value:X8}";
+}