From e40e06896de7523c8d5d590a92f787918aa2830d Mon Sep 17 00:00:00 2001 From: dexy Date: Mon, 30 Dec 2019 16:37:29 +1100 Subject: [PATCH] XML/RBF conversion improvements --- CodeWalker.Core/GameFiles/GameFileCache.cs | 118 +++++++++++++++--- .../GameFiles/MetaTypes/MetaXml.cs | 30 ++--- CodeWalker.Core/GameFiles/MetaTypes/Rbf.cs | 18 ++- CodeWalker.Core/GameFiles/MetaTypes/XmlRbf.cs | 84 +++++++++++-- 4 files changed, 204 insertions(+), 46 deletions(-) diff --git a/CodeWalker.Core/GameFiles/GameFileCache.cs b/CodeWalker.Core/GameFiles/GameFileCache.cs index 2d22f88..84ee29c 100644 --- a/CodeWalker.Core/GameFiles/GameFileCache.cs +++ b/CodeWalker.Core/GameFiles/GameFileCache.cs @@ -182,6 +182,7 @@ namespace CodeWalker.GameFiles //TestAudioYmts(); //TestMetas(); //TestPsos(); + //TestRbfs(); //TestCuts(); //TestYlds(); //TestYcds(); @@ -476,11 +477,11 @@ namespace CodeWalker.GameFiles //filter ActiveMapFiles based on images.meta? - //DlcContentDataFile imagesdata; - //if (imagedatafiles.TryGetValue(path, out imagesdata)) - //{ - // ActiveMapRpfFiles[path] = baserpf; - //} + //DlcContentDataFile imagesdata; + //if (imagedatafiles.TryGetValue(path, out imagesdata)) + //{ + // ActiveMapRpfFiles[path] = baserpf; + //} } private void InitActiveMapRpfFiles() @@ -534,9 +535,9 @@ namespace CodeWalker.GameFiles Dictionary> overlays = new Dictionary>(); - foreach(var setupfile in DlcSetupFiles) - { - if(setupfile.DlcFile!=null) + foreach (var setupfile in DlcSetupFiles) + { + if (setupfile.DlcFile != null) { //if (setupfile.order > maxdlcorder) // break; @@ -628,7 +629,7 @@ namespace CodeWalker.GameFiles else if (contentfile.RpfDataFiles.TryGetValue(dfn, out rpfdatafile)) { string phpath = GetDlcRpfPhysicalPath(rpfdatafile.filename, setupfile); - + //if (rpfdatafile.overlay) AddDlcOverlayRpf(dfn, rpfdatafile.filename, setupfile, overlays); @@ -715,7 +716,7 @@ namespace CodeWalker.GameFiles else { } //how to handle individual files? eg interiorProxies.meta } - private void AddDlcOverlayRpf(string path, string umpath, DlcSetupFile setupfile, Dictionary> overlays) + private void AddDlcOverlayRpf(string path, string umpath, DlcSetupFile setupfile, Dictionary> overlays) { string opath = GetDlcOverlayPath(umpath, setupfile); if (opath == path) return; @@ -1079,7 +1080,7 @@ namespace CodeWalker.GameFiles IEnumerable rpfs = PreloadedMode ? AllRpfs : (IEnumerable)ActiveMapRpfFiles.Values; - var addTxdRelationships = new Action>((from) => + var addTxdRelationships = new Action>((from) => { foreach (var kvp in from) { @@ -1516,7 +1517,7 @@ namespace CodeWalker.GameFiles - IEnumerable rpfs = PreloadedMode ? AllRpfs : (IEnumerable)ActiveMapRpfFiles.Values; + IEnumerable rpfs = PreloadedMode ? AllRpfs : (IEnumerable)ActiveMapRpfFiles.Values; var allVehicles = new Dictionary(); @@ -1655,7 +1656,7 @@ namespace CodeWalker.GameFiles return dict; } - var addPedDicts = new Action((namel, hash, dir)=> + var addPedDicts = new Action((namel, hash, dir) => { Dictionary dict = null; var files = dir?.Files; @@ -1963,7 +1964,7 @@ namespace CodeWalker.GameFiles //ErrorLog("Drawable dictionary not found: " + JenkIndex.GetString(hash)); //too spammy... } } - else if(!ydd.Loaded) + else if (!ydd.Loaded) { TryLoadEnqueue(ydd); } @@ -2236,7 +2237,7 @@ namespace CodeWalker.GameFiles - public bool LoadFile(T file) where T:GameFile,PackedFile + public bool LoadFile(T file) where T : GameFile, PackedFile { if (file == null) return false; RpfFileEntry entry = file.RpfFileEntry; @@ -2384,7 +2385,7 @@ namespace CodeWalker.GameFiles public YtdFile TryGetParentYtd(uint hash) { MetaHash phash; - if(textureParents.TryGetValue(hash, out phash)) + if (textureParents.TryGetValue(hash, out phash)) { return GetYtd(phash); } @@ -2890,7 +2891,7 @@ namespace CodeWalker.GameFiles if (xml.Length != xml2.Length) { } - if ((xml != xml2)&&(!n.EndsWith("srl.ymt") && !n.StartsWith("des_"))) + if ((xml != xml2) && (!n.EndsWith("srl.ymt") && !n.StartsWith("des_"))) { } } @@ -2925,7 +2926,7 @@ namespace CodeWalker.GameFiles #endif { var n = entry.NameLower; - if (!(n.EndsWith(".pso") || + if (!(n.EndsWith(".pso") || n.EndsWith(".ymt") || n.EndsWith(".ymf") || n.EndsWith(".ymap") || @@ -3008,6 +3009,87 @@ namespace CodeWalker.GameFiles { } } + public void TestRbfs() + { + var exceptions = new List(); + var allrbfs = new List(); + var diffrbfs = new List(); + + foreach (RpfFile file in AllRpfs) + { + foreach (RpfEntry entry in file.AllEntries) + { + var n = entry.NameLower; + if (!(n.EndsWith(".ymt") || + n.EndsWith(".ymf") || + n.EndsWith(".ymap") || + n.EndsWith(".ytyp") || + n.EndsWith(".cut"))) + continue; //PSO files seem to only have these extensions + + var fentry = entry as RpfFileEntry; + var data = entry.File.ExtractFile(fentry); + if (data != null) + { + using (MemoryStream ms = new MemoryStream(data)) + { + if (RbfFile.IsRBF(ms)) + { + UpdateStatus(string.Format(entry.Path)); + + var rbf = new RbfFile(); + rbf.Load(ms); + + allrbfs.Add(fentry.Path); + + var xml = RbfXml.GetXml(rbf); + if (!string.IsNullOrEmpty(xml)) + { } + + var xdoc = new XmlDocument(); + xdoc.LoadXml(xml); + var rbf2 = XmlRbf.GetRbf(xdoc); + var rbf2b = rbf2.Save(); + + var rbf3 = new RbfFile(); + rbf3.Load(rbf2b); + var xml3 = RbfXml.GetXml(rbf3); + + if (xml.Length != xml3.Length) + { } + if (xml != xml3) + { + diffrbfs.Add(fentry.Path); + } + + if (data.Length != rbf2b.Length) + { + //File.WriteAllBytes("C:\\GitHub\\CodeWalkerResearch\\RBF\\" + fentry.Name + ".dat0", data); + //File.WriteAllBytes("C:\\GitHub\\CodeWalkerResearch\\RBF\\" + fentry.Name + ".dat1", rbf2b); + } + else + { + for (int i = 0; i < data.Length; i++) + { + if (data[i] != rbf2b[i]) + { + diffrbfs.Add(fentry.Path); + break; + } + } + } + + } + } + } + + } + } + + string allrbfpaths = string.Join("\r\n", allrbfs); + string diffrbfpaths = string.Join("\r\n", diffrbfs); + + } public void TestCuts() { diff --git a/CodeWalker.Core/GameFiles/MetaTypes/MetaXml.cs b/CodeWalker.Core/GameFiles/MetaTypes/MetaXml.cs index 77c5ecc..1525fdd 100644 --- a/CodeWalker.Core/GameFiles/MetaTypes/MetaXml.cs +++ b/CodeWalker.Core/GameFiles/MetaTypes/MetaXml.cs @@ -1553,7 +1553,7 @@ namespace CodeWalker.GameFiles int cind = indent + 1; - bool oneline = ((rs.Children.Count == 1) && (rs.Children[0].Name == null)); + bool oneline = ((rs.Children.Count == 1) && (rs.Children[0].Name == null) && (rs.Attributes.Count == 0)); OpenTag(sb, indent, rs.Name + attStr, !oneline); @@ -1563,17 +1563,14 @@ namespace CodeWalker.GameFiles if (child is RbfBytes) { var bytesChild = (RbfBytes)child; - var contentField = rs.FindChild("content") as RbfString;//TODO: fix this to output nicer XML! + var contentField = rs.FindAttribute("content") as RbfString;//TODO: fix this to output nicer XML! if (contentField != null) { - OpenTag(sb, cind, "value"); - var aind = cind + 1; - if (contentField.Value == "char_array") { foreach (byte k in bytesChild.Value) { - Indent(sb, aind); + Indent(sb, cind); sb.AppendLine(k.ToString()); } } @@ -1582,23 +1579,20 @@ namespace CodeWalker.GameFiles var valueReader = new DataReader(new MemoryStream(bytesChild.Value)); while (valueReader.Position < valueReader.Length) { - Indent(sb, aind); + Indent(sb, cind); var y = valueReader.ReadUInt16(); sb.AppendLine(y.ToString()); } } else { - ErrorXml(sb, aind, "Unexpected content type: " + contentField.Value); + ErrorXml(sb, cind, "Unexpected content type: " + contentField.Value); } - - CloseTag(sb, cind, "value"); } else { string stringValue = Encoding.ASCII.GetString(bytesChild.Value); string str = stringValue.Substring(0, stringValue.Length - 1); //removes null terminator - sb.Append(str); } } @@ -1609,6 +1603,8 @@ namespace CodeWalker.GameFiles } if (child is RbfString) { + ////// this doesn't seem to be used! it's always using RbfBytes child... + var stringChild = (RbfString)child; StringTag(sb, cind, stringChild.Name, stringChild.Value); @@ -1940,14 +1936,14 @@ namespace CodeWalker.GameFiles public static string UintString(uint h) { - string str; - if (MetaNames.TryGetString(h, out str)) return str; + //string str; + //if (MetaNames.TryGetString(h, out str)) return str; - str = JenkIndex.TryGetString(h); - if (!string.IsNullOrEmpty(str)) return str; + //str = JenkIndex.TryGetString(h); + //if (!string.IsNullOrEmpty(str)) return str; - //TODO: do extra hash lookup here - //if(Lookup.TryGetValue(uh, out str)) ... + ////TODO: do extra hash lookup here + ////if(Lookup.TryGetValue(uh, out str)) ... //if (h == 0) return ""; diff --git a/CodeWalker.Core/GameFiles/MetaTypes/Rbf.cs b/CodeWalker.Core/GameFiles/MetaTypes/Rbf.cs index 4469ec0..f2f2e55 100644 --- a/CodeWalker.Core/GameFiles/MetaTypes/Rbf.cs +++ b/CodeWalker.Core/GameFiles/MetaTypes/Rbf.cs @@ -44,6 +44,13 @@ namespace CodeWalker.GameFiles public List descriptors { get; set; } public Dictionary outDescriptors { get; private set; } = new Dictionary(); + + public void Load(byte[] data) + { + using (var ms = new MemoryStream(data)) + Load(ms); + } + public RbfStructure Load(string fileName) { using (var fileStream = new FileStream(fileName, FileMode.Open)) @@ -222,7 +229,7 @@ namespace CodeWalker.GameFiles public byte GetDescriptorIndex(IRbfType t, out bool isNew) { - var key = $"{t.Name}_{t.DataType}"; + var key = t.Name;// $"{t.Name}_{t.DataType}"; isNew = false; if (!outDescriptors.TryGetValue(key, out var idx)) @@ -388,6 +395,15 @@ namespace CodeWalker.GameFiles } return null; } + public IRbfType FindAttribute(string name) + { + foreach (var attr in Attributes) + { + if (attr == null) continue; + if (attr.Name == name) return attr; + } + return null; + } public void Save(RbfFile root, DataWriter writer) { root.WriteRecordId(this, writer); diff --git a/CodeWalker.Core/GameFiles/MetaTypes/XmlRbf.cs b/CodeWalker.Core/GameFiles/MetaTypes/XmlRbf.cs index 3265777..49db172 100644 --- a/CodeWalker.Core/GameFiles/MetaTypes/XmlRbf.cs +++ b/CodeWalker.Core/GameFiles/MetaTypes/XmlRbf.cs @@ -56,13 +56,15 @@ namespace CodeWalker.GameFiles Z = z }; } - else if ((element.Elements().Count() == 0) && (element.Attributes().Count() == 0)) //else if (element.Name == "type" || element.Name == "key" || element.Name == "platform") + else if ((element.Elements().Count() == 0) && (element.Attributes().Count() == 0) && (!element.IsEmpty)) //else if (element.Name == "type" || element.Name == "key" || element.Name == "platform") { - return new RbfString() - { - Name = element.Name.LocalName, - Value = element.Value - }; + var bytearr = Encoding.ASCII.GetBytes(element.Value); + var bytearrnt = new byte[bytearr.Length + 1]; + Buffer.BlockCopy(bytearr, 0, bytearrnt, 0, bytearr.Length); + var bytes = new RbfBytes() { Value = bytearrnt }; + var struc = new RbfStructure() { Name = element.Name.LocalName }; + struc.Children.Add(bytes); + return struc; } var n = new RbfStructure(); @@ -83,11 +85,33 @@ namespace CodeWalker.GameFiles } else if (node is XText text) { - return new RbfBytes() + byte[] bytes = null; + var contentAttr = node.Parent?.Attribute("content"); + if (contentAttr != null) { - Name = "", - Value = Encoding.ASCII.GetBytes(text.Value).Concat(new byte[] { 0x00 }).ToArray() - }; + if (contentAttr.Value == "char_array") + { + bytes = GetByteArray(text.Value); + } + else if (contentAttr.Value == "short_array") + { + bytes = GetUshortArray(text.Value); + } + else + { } + } + else + { + bytes = Encoding.ASCII.GetBytes(text.Value).Concat(new byte[] { 0x00 }).ToArray(); + } + if (bytes != null) + { + return new RbfBytes() + { + Name = "", + Value = bytes + }; + } } return null; @@ -138,5 +162,45 @@ namespace CodeWalker.GameFiles }; } } + + + + + + private static byte[] GetByteArray(string text) + { + if (string.IsNullOrEmpty(text)) return null; + var data = new List(); + var split = Regex.Split(text, @"[\s\r\n\t]"); + for (int i = 0; i < split.Length; i++) + { + if (!string.IsNullOrEmpty(split[i])) + { + var str = split[i]; + if (string.IsNullOrEmpty(str)) continue; + var val = Convert.ToByte(str); + data.Add(val); + } + } + return data.ToArray(); + } + private static byte[] GetUshortArray(string text) + { + var data = new List(); + var split = Regex.Split(text, @"[\s\r\n\t]"); + for (int i = 0; i < split.Length; i++) + { + if (!string.IsNullOrEmpty(split[i])) + { + var str = split[i]; + if (string.IsNullOrEmpty(str)) continue; + var val = Convert.ToUInt16(str); + data.Add((byte)((val >> 0) & 0xFF)); + data.Add((byte)((val >> 8) & 0xFF)); + } + } + return data.ToArray(); + } + } }