From 65a3435fb9cb60786355214f6096860120eff0a7 Mon Sep 17 00:00:00 2001 From: dexy Date: Thu, 12 Mar 2020 00:05:49 +1100 Subject: [PATCH] TextureBase BlockLength fix, ResourceAnalyzer improvement, resource packing improvement --- .../GameFiles/Resources/Drawable.cs | 13 +- CodeWalker.Core/GameFiles/Resources/Frag.cs | 9 +- .../GameFiles/Resources/ResourceAnalyzer.cs | 62 ++++- .../GameFiles/Resources/ResourceBaseTypes.cs | 13 - .../GameFiles/Resources/ResourceBuilder.cs | 2 +- .../GameFiles/Resources/ResourceData.cs | 23 +- .../GameFiles/Resources/Texture.cs | 235 ++++++++++-------- 7 files changed, 224 insertions(+), 133 deletions(-) diff --git a/CodeWalker.Core/GameFiles/Resources/Drawable.cs b/CodeWalker.Core/GameFiles/Resources/Drawable.cs index ba2f858..b07f051 100644 --- a/CodeWalker.Core/GameFiles/Resources/Drawable.cs +++ b/CodeWalker.Core/GameFiles/Resources/Drawable.cs @@ -543,6 +543,17 @@ namespace CodeWalker.GameFiles Hashes = hashes.ToArray(); + //####### TODO: investigate missing data here!!! ####### + //var bytes = reader.ReadBytes(142 * Count); + //if (bytes != null) + //{ + // for (int i = 0; i < bytes.Length; i++) + // { + // if (bytes[i] != 0) + // { break; } + // } + //} + //// just testing... //for (int i = 0; i < Parameters.Length; i++) //{ @@ -661,7 +672,6 @@ namespace CodeWalker.GameFiles { YdrXml.OpenTag(sb, indent, otstr); YdrXml.StringTag(sb, cind, "Name", YdrXml.XmlEscape(tex.Name)); - YdrXml.ValueTag(sb, cind, "Unk32", tex.Unknown_32h.ToString()); YdrXml.CloseTag(sb, indent, "Item"); } else @@ -715,6 +725,7 @@ namespace CodeWalker.GameFiles { var tex = new TextureBase(); tex.ReadXml(pnode, null);//embedded textures will get replaced in ShaderFX ReadXML + tex.Unknown_32h = 2; p.Data = tex; } } diff --git a/CodeWalker.Core/GameFiles/Resources/Frag.cs b/CodeWalker.Core/GameFiles/Resources/Frag.cs index c70785d..f31170e 100644 --- a/CodeWalker.Core/GameFiles/Resources/Frag.cs +++ b/CodeWalker.Core/GameFiles/Resources/Frag.cs @@ -106,7 +106,9 @@ namespace CodeWalker.GameFiles public YftFile Yft { get; set; } - //public ResourceAnalyzer Analyzer { get; set; } +#if DEBUG + public ResourceAnalyzer Analyzer { get; set; } +#endif public override void Read(ResourceDataReader reader, params object[] parameters) { @@ -197,8 +199,9 @@ namespace CodeWalker.GameFiles AssignChildrenShaders(); - //Analyzer = new ResourceAnalyzer(reader); - +#if DEBUG + Analyzer = new ResourceAnalyzer(reader); +#endif ////just testing!! //if (BoundingSphereRadius <= 0.0f) diff --git a/CodeWalker.Core/GameFiles/Resources/ResourceAnalyzer.cs b/CodeWalker.Core/GameFiles/Resources/ResourceAnalyzer.cs index b0a6246..be0f9fa 100644 --- a/CodeWalker.Core/GameFiles/Resources/ResourceAnalyzer.cs +++ b/CodeWalker.Core/GameFiles/Resources/ResourceAnalyzer.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Runtime.InteropServices; using System.Text; namespace CodeWalker.GameFiles @@ -17,12 +18,15 @@ namespace CodeWalker.GameFiles public long Position { get; set; } public long Length { get; set; } public long Offset { get { return Position & 0xFFFFFFF; } } + public bool Overlapping { get; set; } public ResourceSystemBlock SystemBlock { get; set; } public ResourceGraphicsBlock GraphicsBlock { get; set; } + public Array Array { get; set; } + public override string ToString() { - var type = ""; + var type = "########## ??? ##########"; if (SystemBlock != null) { type = SystemBlock.GetType().Name; @@ -31,15 +35,18 @@ namespace CodeWalker.GameFiles { type = GraphicsBlock.GetType().Name; } - return Offset.ToString() + " - " + Length.ToString() + " - " + type; + if (Array != null) + { + type = Array.GetType().Name + " (" + Array.Length.ToString() + ")"; + } + return Offset.ToString() + " - " + Length.ToString() + " - " + type + (Overlapping ? " (embedded)" : ""); } } public ResourceAnalyzer(ResourceDataReader reader) { var dlist = new List(); - var dict = reader.blockPool; - foreach (var kvp in dict) + foreach (var kvp in reader.blockPool) { var item = new ResourceAnalyzerItem(); item.Position = kvp.Key; @@ -48,11 +55,56 @@ namespace CodeWalker.GameFiles item.GraphicsBlock = kvp.Value as ResourceGraphicsBlock; dlist.Add(item); } + foreach (var kvp in reader.arrayPool) + { + var item = new ResourceAnalyzerItem(); + item.Position = kvp.Key; + item.Array = kvp.Value as Array; + if (item.Array != null) + { + var typ = item.Array.GetType().GetElementType(); + var siz = Marshal.SizeOf(typ); + item.Length = item.Array.Length * siz; + } + dlist.Add(item); + } dlist.Sort((a, b) => a.Position.CompareTo(b.Position)); - Blocks = dlist.ToArray(); + //Blocks = dlist.ToArray(); + + + var dlist2 = new List(); + long pos = 0; + foreach (var item in dlist) + { + if (item.Offset > pos) + { + var gap = new ResourceAnalyzerItem(); + gap.Position = pos; + gap.Length = item.Offset - pos; + dlist2.Add(gap); + pos = item.Offset; + } + if (item.Offset == pos) + { + dlist2.Add(item); + pos = item.Offset + item.Length; + if ((pos % 16) != 0) pos += (16 - (pos % 16));//ignore alignment paddings + } + else + { + item.Overlapping = true; + dlist2.Add(item); + var pos2 = item.Offset + item.Length; + if (pos2 > pos) pos = pos2; + } + } + + + Blocks = dlist2.ToArray(); } + } } diff --git a/CodeWalker.Core/GameFiles/Resources/ResourceBaseTypes.cs b/CodeWalker.Core/GameFiles/Resources/ResourceBaseTypes.cs index acd8370..aed3b11 100644 --- a/CodeWalker.Core/GameFiles/Resources/ResourceBaseTypes.cs +++ b/CodeWalker.Core/GameFiles/Resources/ResourceBaseTypes.cs @@ -1362,19 +1362,6 @@ namespace CodeWalker.GameFiles return "(Count: " + EntriesCount.ToString() + ")"; } } - [TypeConverter(typeof(ExpandableObjectConverter))] public struct ResourceSimpleList64Ptr - { - // structure data - public ulong EntriesPointer { get; private set; } - public ushort EntriesCount { get; private set; } - public ushort EntriesCapacity { get; private set; } - public uint Unused1 { get; private set; } - - public override string ToString() - { - return "(Count: " + EntriesCount.ToString() + ")"; - } - } [TypeConverter(typeof(ExpandableObjectConverter))] public class ResourcePointerArray64 : ResourceSystemBlock, IList where T : IResourceSystemBlock, new() diff --git a/CodeWalker.Core/GameFiles/Resources/ResourceBuilder.cs b/CodeWalker.Core/GameFiles/Resources/ResourceBuilder.cs index d8d2dbc..38316b4 100644 --- a/CodeWalker.Core/GameFiles/Resources/ResourceBuilder.cs +++ b/CodeWalker.Core/GameFiles/Resources/ResourceBuilder.cs @@ -244,7 +244,7 @@ namespace CodeWalker.GameFiles { block.FilePosition = basePosition + currentPosition; - currentPosition += block.BlockLength + SKIP_SIZE; + currentPosition += block.BlockLength; if ((currentPosition % ALIGN_SIZE) != 0) { diff --git a/CodeWalker.Core/GameFiles/Resources/ResourceData.cs b/CodeWalker.Core/GameFiles/Resources/ResourceData.cs index e4a6a21..4a577dc 100644 --- a/CodeWalker.Core/GameFiles/Resources/ResourceData.cs +++ b/CodeWalker.Core/GameFiles/Resources/ResourceData.cs @@ -51,6 +51,7 @@ namespace CodeWalker.GameFiles // this is a dictionary that contains all the resource blocks // which were read from this resource reader public Dictionary blockPool = new Dictionary(); + public Dictionary arrayPool = new Dictionary(); /// /// Gets the length of the underlying stream. @@ -207,9 +208,7 @@ namespace CodeWalker.GameFiles blockPool[Position] = result; } - var classPosition = Position; result.Read(this, parameters); - //result.Position = classPosition; //TODO: need this if writing stuff! return result; } @@ -244,6 +243,7 @@ namespace CodeWalker.GameFiles Position = pos; var result = ReadBytes((int)count); Position = posbackup; + arrayPool[(long)position] = result; return result; } public ushort[] ReadUshortsAt(ulong position, uint count) @@ -263,6 +263,9 @@ namespace CodeWalker.GameFiles // result2[i] = ReadUInt16(); //} //Position = posbackup; + + arrayPool[(long)position] = result; + return result; } public short[] ReadShortsAt(ulong position, uint count) @@ -272,6 +275,9 @@ namespace CodeWalker.GameFiles var length = count * 2; byte[] data = ReadBytesAt(position, length); Buffer.BlockCopy(data, 0, result, 0, (int)length); + + arrayPool[(long)position] = result; + return result; } public uint[] ReadUintsAt(ulong position, uint count) @@ -291,6 +297,9 @@ namespace CodeWalker.GameFiles // result[i] = ReadUInt32(); //} //Position = posbackup; + + arrayPool[(long)position] = result; + return result; } public ulong[] ReadUlongsAt(ulong position, uint count) @@ -310,6 +319,9 @@ namespace CodeWalker.GameFiles // result[i] = ReadUInt64(); //} //Position = posbackup; + + arrayPool[(long)position] = result; + return result; } public float[] ReadFloatsAt(ulong position, uint count) @@ -329,6 +341,9 @@ namespace CodeWalker.GameFiles // result[i] = ReadSingle(); //} //Position = posbackup; + + arrayPool[(long)position] = result; + return result; } public T[] ReadStructsAt(ulong position, uint count)//, uint structsize) @@ -351,6 +366,8 @@ namespace CodeWalker.GameFiles } handle.Free(); + arrayPool[(long)position] = result; + return result; } public T[] ReadStructs(uint count) @@ -592,7 +609,7 @@ namespace CodeWalker.GameFiles /// - /// Represents a data block that won't get cached while loading. Used for 0-length object parts + /// Represents a data block that won't get cached while loading. /// public interface IResourceNoCacheBlock : IResourceBlock { } diff --git a/CodeWalker.Core/GameFiles/Resources/Texture.cs b/CodeWalker.Core/GameFiles/Resources/Texture.cs index 61dadca..3920334 100644 --- a/CodeWalker.Core/GameFiles/Resources/Texture.cs +++ b/CodeWalker.Core/GameFiles/Resources/Texture.cs @@ -181,7 +181,7 @@ namespace CodeWalker.GameFiles { public override long BlockLength { - get { return 64; } + get { return 80; } } // structure data @@ -201,6 +201,10 @@ namespace CodeWalker.GameFiles public uint Unknown_34h { get; set; } // 0x00000000 public uint Unknown_38h { get; set; } // 0x00000000 public uint Unknown_3Ch { get; set; } // 0x00000000 + public uint UsageData { get; set; } + public uint Unknown_44h { get; set; } // 0x00000000 + public uint ExtraFlags { get; set; } // 0, 1 + public uint Unknown_4Ch { get; set; } // 0x00000000 // reference data public string Name { get; set; } @@ -208,6 +212,33 @@ namespace CodeWalker.GameFiles private string_r NameBlock = null; + + + public TextureUsage Usage + { + get + { + return (TextureUsage)(UsageData & 0x1F); + } + set + { + UsageData = (UsageData & 0xFFFFFFE0) + (((uint)value) & 0x1F); + } + } + public TextureUsageFlags UsageFlags + { + get + { + return (TextureUsageFlags)(UsageData >> 5); + } + set + { + UsageData = (UsageData & 0x1F) + (((uint)value) << 5); + } + } + + + public override void Read(ResourceDataReader reader, params object[] parameters) { // read structure data @@ -227,6 +258,12 @@ namespace CodeWalker.GameFiles this.Unknown_34h = reader.ReadUInt32(); this.Unknown_38h = reader.ReadUInt32(); this.Unknown_3Ch = reader.ReadUInt32(); + this.UsageData = reader.ReadUInt32(); + this.Unknown_44h = reader.ReadUInt32(); + this.ExtraFlags = reader.ReadUInt32(); + this.Unknown_4Ch = reader.ReadUInt32(); + + // read reference data this.Name = reader.ReadStringAt( //BlockAt( @@ -238,21 +275,86 @@ namespace CodeWalker.GameFiles NameHash = JenkHash.GenHash(Name.ToLowerInvariant()); } - switch (Unknown_32h) - { - case 0x20: - case 0x28: - case 0x30: - case 0x38: - case 0x40: - case 0x48: - case 0x80: - case 0x90: - case 0x2://embedded - break; - default: - break; - } + //switch (Unknown_32h) + //{ + // case 0x20: + // case 0x28: + // case 0x30: + // case 0x38: + // case 0x40: + // case 0x48: + // case 0x80: + // case 0x90: + // case 0x2://base/shaderparam + // break; + // default: + // break;//no hit + //} + + //switch (Usage) + //{ + // case TextureUsage.UNKNOWN:// = 0, + // case TextureUsage.DEFAULT:// = 1, + // case TextureUsage.TERRAIN:// = 2, + // case TextureUsage.CLOUDDENSITY:// = 3, + // case TextureUsage.CLOUDNORMAL:// = 4, + // case TextureUsage.CABLE:// = 5, + // case TextureUsage.FENCE:// = 6, + // case TextureUsage.SCRIPT:// = 8, + // case TextureUsage.WATERFLOW:// = 9, + // case TextureUsage.WATERFOAM:// = 10, + // case TextureUsage.WATERFOG:// = 11, + // case TextureUsage.WATEROCEAN:// = 12, + // case TextureUsage.FOAMOPACITY:// = 14, + // case TextureUsage.DIFFUSEMIPSHARPEN:// = 16, + // case TextureUsage.DIFFUSEDARK:// = 18, + // case TextureUsage.DIFFUSEALPHAOPAQUE:// = 19, + // case TextureUsage.DIFFUSE:// = 20, + // case TextureUsage.DETAIL:// = 21, + // case TextureUsage.NORMAL:// = 22, + // case TextureUsage.SPECULAR:// = 23, + // case TextureUsage.EMISSIVE:// = 24, + // case TextureUsage.TINTPALETTE:// = 25, + // case TextureUsage.SKIPPROCESSING:// = 26, + // break; + // case TextureUsage.ENVEFF:// = 7, //unused by V + // case TextureUsage.WATER:// = 13, //unused by V + // case TextureUsage.FOAM:// = 15, //unused by V + // case TextureUsage.DIFFUSEDETAIL:// = 17, //unused by V + // case TextureUsage.DONOTOPTIMIZE:// = 27, //unused by V + // case TextureUsage.TEST:// = 28, //unused by V + // case TextureUsage.COUNT:// = 29, //unused by V + // break;//no hit + // default: + // break;//no hit + //} + + //var uf = UsageFlags; + //if ((uf & TextureUsageFlags.EMBEDDEDSCRIPTRT) > 0) // .ydr embedded script_rt textures, only 3 uses + //{ } + //if ((uf & TextureUsageFlags.UNK19) > 0) + //{ }//no hit + //if ((uf & TextureUsageFlags.UNK20) > 0) + //{ }//no hit + //if ((uf & TextureUsageFlags.UNK21) > 0) + //{ }//no hit + //if ((uf & TextureUsageFlags.UNK24) == 0)//wtf isthis? only 0 on special resident(?) textures and some reused ones + //{ } + + //if (!(this is Texture)) + //{ + // if (Unknown_32h != 0x2)//base/shaderparam + // { }//no hit + // if (UsageData != 0) + // { }//no hit + // if (Unknown_44h != 0) + // { }//no hit + // if (ExtraFlags != 0) + // { }//no hit + // if (Unknown_4Ch != 0) + // { }//no hit + //} + } public override void Write(ResourceDataWriter writer, params object[] parameters) { @@ -276,17 +378,27 @@ namespace CodeWalker.GameFiles writer.Write(this.Unknown_34h); writer.Write(this.Unknown_38h); writer.Write(this.Unknown_3Ch); + writer.Write(this.UsageData); + writer.Write(this.Unknown_44h); + writer.Write(this.ExtraFlags); + writer.Write(this.Unknown_4Ch); } public virtual void WriteXml(StringBuilder sb, int indent, string ddsfolder) { YtdXml.StringTag(sb, indent, "Name", YtdXml.XmlEscape(Name)); YtdXml.ValueTag(sb, indent, "Unk32", Unknown_32h.ToString()); + YtdXml.StringTag(sb, indent, "Usage", Usage.ToString()); + YtdXml.StringTag(sb, indent, "UsageFlags", UsageFlags.ToString()); + YtdXml.ValueTag(sb, indent, "ExtraFlags", ExtraFlags.ToString()); } public virtual void ReadXml(XmlNode node, string ddsfolder) { Name = Xml.GetChildInnerText(node, "Name"); NameHash = JenkHash.GenHash(Name?.ToLowerInvariant()); Unknown_32h = (ushort)Xml.GetChildUIntAttribute(node, "Unk32", "value"); + Usage = Xml.GetChildEnumInnerText(node, "Usage"); + UsageFlags = Xml.GetChildEnumInnerText(node, "UsageFlags"); + ExtraFlags = Xml.GetChildUIntAttribute(node, "ExtraFlags", "value"); } @@ -315,10 +427,6 @@ namespace CodeWalker.GameFiles } // structure data - public uint UsageData { get; set; } - public uint Unknown_44h { get; set; } // 0x00000000 - public uint ExtraFlags { get; set; } // 0, 1 - public uint Unknown_4Ch { get; set; } // 0x00000000 public ushort Width { get; set; } public ushort Height { get; set; } public ushort Depth { get; set; } = 1; //is depth > 1 supported? @@ -339,29 +447,6 @@ namespace CodeWalker.GameFiles public uint Unknown_88h { get; set; } // 0x00000000 public uint Unknown_8Ch { get; set; } // 0x00000000 - public TextureUsage Usage - { - get - { - return (TextureUsage)(UsageData & 0x1F); - } - set - { - UsageData = (UsageData & 0xFFFFFFE0) + (((uint)value) & 0x1F); - } - } - public TextureUsageFlags UsageFlags - { - get - { - return (TextureUsageFlags)(UsageData >> 5); - } - set - { - UsageData = (UsageData & 0x1F) + (((uint)value) << 5); - } - } - // reference data public TextureData Data { get; set; } @@ -384,10 +469,6 @@ namespace CodeWalker.GameFiles base.Read(reader, parameters); // read structure data - this.UsageData = reader.ReadUInt32(); - this.Unknown_44h = reader.ReadUInt32(); - this.ExtraFlags = reader.ReadUInt32(); - this.Unknown_4Ch = reader.ReadUInt32(); this.Width = reader.ReadUInt16(); this.Height = reader.ReadUInt16(); this.Depth = reader.ReadUInt16(); @@ -411,56 +492,6 @@ namespace CodeWalker.GameFiles // read reference data this.Data = reader.ReadBlockAt(this.DataPointer, this.Format, this.Width, this.Height, this.Levels, this.Stride); - - switch (Usage) - { - case TextureUsage.UNKNOWN:// = 0, - case TextureUsage.DEFAULT:// = 1, - case TextureUsage.TERRAIN:// = 2, - case TextureUsage.CLOUDDENSITY:// = 3, - case TextureUsage.CLOUDNORMAL:// = 4, - case TextureUsage.CABLE:// = 5, - case TextureUsage.FENCE:// = 6, - case TextureUsage.SCRIPT:// = 8, - case TextureUsage.WATERFLOW:// = 9, - case TextureUsage.WATERFOAM:// = 10, - case TextureUsage.WATERFOG:// = 11, - case TextureUsage.WATEROCEAN:// = 12, - case TextureUsage.FOAMOPACITY:// = 14, - case TextureUsage.DIFFUSEMIPSHARPEN:// = 16, - case TextureUsage.DIFFUSEDARK:// = 18, - case TextureUsage.DIFFUSEALPHAOPAQUE:// = 19, - case TextureUsage.DIFFUSE:// = 20, - case TextureUsage.DETAIL:// = 21, - case TextureUsage.NORMAL:// = 22, - case TextureUsage.SPECULAR:// = 23, - case TextureUsage.EMISSIVE:// = 24, - case TextureUsage.TINTPALETTE:// = 25, - case TextureUsage.SKIPPROCESSING:// = 26, - break; - case TextureUsage.ENVEFF:// = 7, //unused by V - case TextureUsage.WATER:// = 13, //unused by V - case TextureUsage.FOAM:// = 15, //unused by V - case TextureUsage.DIFFUSEDETAIL:// = 17, //unused by V - case TextureUsage.DONOTOPTIMIZE:// = 27, //unused by V - case TextureUsage.TEST:// = 28, //unused by V - case TextureUsage.COUNT:// = 29, //unused by V - break; - default: - break; - } - - var uf = UsageFlags; - if ((uf & TextureUsageFlags.EMBEDDEDSCRIPTRT) > 0) // .ydr embedded script_rt textures, only 3 uses - { } - if ((uf & TextureUsageFlags.UNK19) > 0) - { } - if ((uf & TextureUsageFlags.UNK20) > 0) - { } - if ((uf & TextureUsageFlags.UNK21) > 0) - { } - if ((uf & TextureUsageFlags.UNK24) == 0)//wtf isthis? only 0 on special resident(?) textures and some reused ones - { } } public override void Write(ResourceDataWriter writer, params object[] parameters) { @@ -469,10 +500,6 @@ namespace CodeWalker.GameFiles this.DataPointer = (ulong)this.Data.FilePosition; // write structure data - writer.Write(this.UsageData); - writer.Write(this.Unknown_44h); - writer.Write(this.ExtraFlags); - writer.Write(this.Unknown_4Ch); writer.Write(this.Width); writer.Write(this.Height); writer.Write(this.Depth); @@ -500,9 +527,6 @@ namespace CodeWalker.GameFiles YtdXml.ValueTag(sb, indent, "Height", Height.ToString()); YtdXml.ValueTag(sb, indent, "MipLevels", Levels.ToString()); YtdXml.StringTag(sb, indent, "Format", Format.ToString()); - YtdXml.StringTag(sb, indent, "Usage", Usage.ToString()); - YtdXml.StringTag(sb, indent, "UsageFlags", UsageFlags.ToString()); - YtdXml.ValueTag(sb, indent, "ExtraFlags", ExtraFlags.ToString()); YtdXml.StringTag(sb, indent, "FileName", YtdXml.XmlEscape((Name ?? "null") + ".dds")); try @@ -527,9 +551,6 @@ namespace CodeWalker.GameFiles Height = (ushort)Xml.GetChildUIntAttribute(node, "Height", "value"); Levels = (byte)Xml.GetChildUIntAttribute(node, "MipLevels", "value"); Format = Xml.GetChildEnumInnerText(node, "Format"); - Usage = Xml.GetChildEnumInnerText(node, "Usage"); - UsageFlags = Xml.GetChildEnumInnerText(node, "UsageFlags"); - ExtraFlags = Xml.GetChildUIntAttribute(node, "ExtraFlags", "value"); var filename = Xml.GetChildInnerText(node, "FileName"); try