XML/RBF conversion improvements

CodeWalker.Core/GameFiles/GameFileCache.cs

@@ -182,6 +182,7 @@ namespace CodeWalker.GameFiles
                 //TestAudioYmts();
                 //TestMetas();
                 //TestPsos();
+                //TestRbfs();
                 //TestCuts();
                 //TestYlds();
                 //TestYcds();
@@ -476,11 +477,11 @@ namespace CodeWalker.GameFiles
 
             //filter ActiveMapFiles based on images.meta?
 
-                        //DlcContentDataFile imagesdata;
+            //DlcContentDataFile imagesdata;
-                        //if (imagedatafiles.TryGetValue(path, out imagesdata))
+            //if (imagedatafiles.TryGetValue(path, out imagesdata))
-                        //{
+            //{
-                        //    ActiveMapRpfFiles[path] = baserpf;
+            //    ActiveMapRpfFiles[path] = baserpf;
-                        //}
+            //}
         }
 
         private void InitActiveMapRpfFiles()
@@ -534,9 +535,9 @@ namespace CodeWalker.GameFiles
 
             Dictionary<string, List<string>> overlays = new Dictionary<string, List<string>>();
 
-            foreach(var setupfile in DlcSetupFiles)
+            foreach (var setupfile in DlcSetupFiles)
-            { 
+            {
-                if(setupfile.DlcFile!=null)
+                if (setupfile.DlcFile != null)
                 {
                     //if (setupfile.order > maxdlcorder)
                     //    break;
@@ -628,7 +629,7 @@ namespace CodeWalker.GameFiles
                                 else if (contentfile.RpfDataFiles.TryGetValue(dfn, out rpfdatafile))
                                 {
                                     string phpath = GetDlcRpfPhysicalPath(rpfdatafile.filename, setupfile);
-                                    
+
                                     //if (rpfdatafile.overlay)
                                     AddDlcOverlayRpf(dfn, rpfdatafile.filename, setupfile, overlays);
 
@@ -715,7 +716,7 @@ namespace CodeWalker.GameFiles
             else
             { } //how to handle individual files? eg interiorProxies.meta
         }
-        private void AddDlcOverlayRpf(string path, string umpath, DlcSetupFile setupfile, Dictionary<string,List<string>> overlays)
+        private void AddDlcOverlayRpf(string path, string umpath, DlcSetupFile setupfile, Dictionary<string, List<string>> overlays)
         {
             string opath = GetDlcOverlayPath(umpath, setupfile);
             if (opath == path) return;
@@ -1079,7 +1080,7 @@ namespace CodeWalker.GameFiles
 
             IEnumerable<RpfFile> rpfs = PreloadedMode ? AllRpfs : (IEnumerable<RpfFile>)ActiveMapRpfFiles.Values;
 
-            var addTxdRelationships = new Action<Dictionary<string, string>>((from) => 
+            var addTxdRelationships = new Action<Dictionary<string, string>>((from) =>
             {
                 foreach (var kvp in from)
                 {
@@ -1516,7 +1517,7 @@ namespace CodeWalker.GameFiles
 
 
 
-            IEnumerable <RpfFile> rpfs = PreloadedMode ? AllRpfs : (IEnumerable<RpfFile>)ActiveMapRpfFiles.Values;
+            IEnumerable<RpfFile> rpfs = PreloadedMode ? AllRpfs : (IEnumerable<RpfFile>)ActiveMapRpfFiles.Values;
 
 
             var allVehicles = new Dictionary<MetaHash, VehicleInitData>();
@@ -1655,7 +1656,7 @@ namespace CodeWalker.GameFiles
                 return dict;
             }
 
-            var addPedDicts = new Action<string, MetaHash, RpfDirectoryEntry>((namel, hash, dir)=>
+            var addPedDicts = new Action<string, MetaHash, RpfDirectoryEntry>((namel, hash, dir) =>
             {
                 Dictionary<MetaHash, RpfFileEntry> dict = null;
                 var files = dir?.Files;
@@ -1963,7 +1964,7 @@ namespace CodeWalker.GameFiles
                         //ErrorLog("Drawable dictionary not found: " + JenkIndex.GetString(hash)); //too spammy...
                     }
                 }
-                else if(!ydd.Loaded)
+                else if (!ydd.Loaded)
                 {
                     TryLoadEnqueue(ydd);
                 }
@@ -2236,7 +2237,7 @@ namespace CodeWalker.GameFiles
 
 
 
-        public bool LoadFile<T>(T file) where T:GameFile,PackedFile
+        public bool LoadFile<T>(T file) where T : GameFile, PackedFile
         {
             if (file == null) return false;
             RpfFileEntry entry = file.RpfFileEntry;
@@ -2384,7 +2385,7 @@ namespace CodeWalker.GameFiles
         public YtdFile TryGetParentYtd(uint hash)
         {
             MetaHash phash;
-            if(textureParents.TryGetValue(hash, out phash))
+            if (textureParents.TryGetValue(hash, out phash))
             {
                 return GetYtd(phash);
             }
@@ -2890,7 +2891,7 @@ namespace CodeWalker.GameFiles
 
                             if (xml.Length != xml2.Length)
                             { }
-                            if ((xml != xml2)&&(!n.EndsWith("srl.ymt") && !n.StartsWith("des_")))
+                            if ((xml != xml2) && (!n.EndsWith("srl.ymt") && !n.StartsWith("des_")))
                             { }
 
                         }
@@ -2925,7 +2926,7 @@ namespace CodeWalker.GameFiles
 #endif
                     {
                         var n = entry.NameLower;
-                        if (!(n.EndsWith(".pso") || 
+                        if (!(n.EndsWith(".pso") ||
                               n.EndsWith(".ymt") ||
                               n.EndsWith(".ymf") ||
                               n.EndsWith(".ymap") ||
@@ -3008,6 +3009,87 @@ namespace CodeWalker.GameFiles
             {
             }
         }
+        public void TestRbfs()
+        {
+            var exceptions = new List<Exception>();
+            var allrbfs = new List<string>();
+            var diffrbfs = new List<string>();
+
+            foreach (RpfFile file in AllRpfs)
+            {
+                foreach (RpfEntry entry in file.AllEntries)
+                {
+                    var n = entry.NameLower;
+                    if (!(n.EndsWith(".ymt") ||
+                          n.EndsWith(".ymf") ||
+                          n.EndsWith(".ymap") ||
+                          n.EndsWith(".ytyp") ||
+                          n.EndsWith(".cut")))
+                        continue; //PSO files seem to only have these extensions
+
+                    var fentry = entry as RpfFileEntry;
+                    var data = entry.File.ExtractFile(fentry);
+                    if (data != null)
+                    {
+                        using (MemoryStream ms = new MemoryStream(data))
+                        {
+                            if (RbfFile.IsRBF(ms))
+                            {
+                                UpdateStatus(string.Format(entry.Path));
+
+                                var rbf = new RbfFile();
+                                rbf.Load(ms);
+
+                                allrbfs.Add(fentry.Path);
+
+                                var xml = RbfXml.GetXml(rbf);
+                                if (!string.IsNullOrEmpty(xml))
+                                { }
+
+                                var xdoc = new XmlDocument();
+                                xdoc.LoadXml(xml);
+                                var rbf2 = XmlRbf.GetRbf(xdoc);
+                                var rbf2b = rbf2.Save();
+
+                                var rbf3 = new RbfFile();
+                                rbf3.Load(rbf2b);
+                                var xml3 = RbfXml.GetXml(rbf3);
+
+                                if (xml.Length != xml3.Length)
+                                { }
+                                if (xml != xml3)
+                                {
+                                    diffrbfs.Add(fentry.Path);
+                                }
+
+                                if (data.Length != rbf2b.Length)
+                                {
+                                    //File.WriteAllBytes("C:\\GitHub\\CodeWalkerResearch\\RBF\\" + fentry.Name + ".dat0", data);
+                                    //File.WriteAllBytes("C:\\GitHub\\CodeWalkerResearch\\RBF\\" + fentry.Name + ".dat1", rbf2b);
+                                }
+                                else
+                                {
+                                    for (int i = 0; i < data.Length; i++)
+                                    {
+                                        if (data[i] != rbf2b[i])
+                                        {
+                                            diffrbfs.Add(fentry.Path);
+                                            break;
+                                        }
+                                    }
+                                }
+
+                            }
+                        }
+                    }
+
+                }
+            }
+
+            string allrbfpaths = string.Join("\r\n", allrbfs);
+            string diffrbfpaths = string.Join("\r\n", diffrbfs);
+
+        }
         public void TestCuts()
         {
 

CodeWalker.Core/GameFiles/MetaTypes/MetaXml.cs

@@ -1553,7 +1553,7 @@ namespace CodeWalker.GameFiles
 
             int cind = indent + 1;
 
-            bool oneline = ((rs.Children.Count == 1) && (rs.Children[0].Name == null));
+            bool oneline = ((rs.Children.Count == 1) && (rs.Children[0].Name == null) && (rs.Attributes.Count == 0));
 
             OpenTag(sb, indent, rs.Name + attStr, !oneline);
 
@@ -1563,17 +1563,14 @@ namespace CodeWalker.GameFiles
                 if (child is RbfBytes)
                 {
                     var bytesChild = (RbfBytes)child;
-                    var contentField = rs.FindChild("content") as RbfString;//TODO: fix this to output nicer XML!
+                    var contentField = rs.FindAttribute("content") as RbfString;//TODO: fix this to output nicer XML!
                     if (contentField != null)
                     {
-                        OpenTag(sb, cind, "value");
-                        var aind = cind + 1;
-
                         if (contentField.Value == "char_array")
                         {
                             foreach (byte k in bytesChild.Value)
                             {
-                                Indent(sb, aind);
+                                Indent(sb, cind);
                                 sb.AppendLine(k.ToString());
                             }
                         }
@@ -1582,23 +1579,20 @@ namespace CodeWalker.GameFiles
                             var valueReader = new DataReader(new MemoryStream(bytesChild.Value));
                             while (valueReader.Position < valueReader.Length)
                             {
-                                Indent(sb, aind);
+                                Indent(sb, cind);
                                 var y = valueReader.ReadUInt16();
                                 sb.AppendLine(y.ToString());
                             }
                         }
                         else
                         {
-                            ErrorXml(sb, aind, "Unexpected content type: " + contentField.Value);
+                            ErrorXml(sb, cind, "Unexpected content type: " + contentField.Value);
                         }
-
-                        CloseTag(sb, cind, "value");
                     }
                     else
                     {
                         string stringValue = Encoding.ASCII.GetString(bytesChild.Value);
                         string str = stringValue.Substring(0, stringValue.Length - 1); //removes null terminator
-
                         sb.Append(str);
                     }
                 }
@@ -1609,6 +1603,8 @@ namespace CodeWalker.GameFiles
                 }
                 if (child is RbfString)
                 {
+                    ////// this doesn't seem to be used! it's always using RbfBytes child...
+
                     var stringChild = (RbfString)child;
                     StringTag(sb, cind, stringChild.Name, stringChild.Value);
 
@@ -1940,14 +1936,14 @@ namespace CodeWalker.GameFiles
 
         public static string UintString(uint h)
         {
-            string str;
+            //string str;
-            if (MetaNames.TryGetString(h, out str)) return str;
+            //if (MetaNames.TryGetString(h, out str)) return str;
 
-            str = JenkIndex.TryGetString(h);
+            //str = JenkIndex.TryGetString(h);
-            if (!string.IsNullOrEmpty(str)) return str;
+            //if (!string.IsNullOrEmpty(str)) return str;
 
-            //TODO: do extra hash lookup here
+            ////TODO: do extra hash lookup here
-            //if(Lookup.TryGetValue(uh, out str)) ...
+            ////if(Lookup.TryGetValue(uh, out str)) ...
 
 
             //if (h == 0) return "";

CodeWalker.Core/GameFiles/MetaTypes/Rbf.cs

@@ -44,6 +44,13 @@ namespace CodeWalker.GameFiles
         public List<RbfEntryDescription> descriptors { get; set; }
         public Dictionary<string, int> outDescriptors { get; private set; } = new Dictionary<string, int>();
 
+
+        public void Load(byte[] data)
+        {
+            using (var ms = new MemoryStream(data))
+                Load(ms);
+        }
+
         public RbfStructure Load(string fileName)
         {
             using (var fileStream = new FileStream(fileName, FileMode.Open))
@@ -222,7 +229,7 @@ namespace CodeWalker.GameFiles
 
         public byte GetDescriptorIndex(IRbfType t, out bool isNew)
         {
-            var key = $"{t.Name}_{t.DataType}";
+            var key = t.Name;// $"{t.Name}_{t.DataType}";
             isNew = false;
 
             if (!outDescriptors.TryGetValue(key, out var idx))
@@ -388,6 +395,15 @@ namespace CodeWalker.GameFiles
             }
             return null;
         }
+        public IRbfType FindAttribute(string name)
+        {
+            foreach (var attr in Attributes)
+            {
+                if (attr == null) continue;
+                if (attr.Name == name) return attr;
+            }
+            return null;
+        }
         public void Save(RbfFile root, DataWriter writer)
         {
             root.WriteRecordId(this, writer);

CodeWalker.Core/GameFiles/MetaTypes/XmlRbf.cs

@@ -56,13 +56,15 @@ namespace CodeWalker.GameFiles
                         Z = z
                     };
                 }
-                else if ((element.Elements().Count() == 0) && (element.Attributes().Count() == 0)) //else if (element.Name == "type" || element.Name == "key" || element.Name == "platform")
+                else if ((element.Elements().Count() == 0) && (element.Attributes().Count() == 0) && (!element.IsEmpty)) //else if (element.Name == "type" || element.Name == "key" || element.Name == "platform")
                 {
-                    return new RbfString()
+                    var bytearr = Encoding.ASCII.GetBytes(element.Value);
-                    {
+                    var bytearrnt = new byte[bytearr.Length + 1];
-                        Name = element.Name.LocalName,
+                    Buffer.BlockCopy(bytearr, 0, bytearrnt, 0, bytearr.Length);
-                        Value = element.Value
+                    var bytes = new RbfBytes() { Value = bytearrnt };
-                    };
+                    var struc = new RbfStructure() { Name = element.Name.LocalName };
+                    struc.Children.Add(bytes);
+                    return struc;
                 }
 
                 var n = new RbfStructure();
@@ -83,11 +85,33 @@ namespace CodeWalker.GameFiles
             }
             else if (node is XText text)
             {
-                return new RbfBytes()
+                byte[] bytes = null;
+                var contentAttr = node.Parent?.Attribute("content");
+                if (contentAttr != null)
                 {
-                    Name = "",
+                    if (contentAttr.Value == "char_array")
-                    Value = Encoding.ASCII.GetBytes(text.Value).Concat(new byte[] { 0x00 }).ToArray()
+                    {
-                };
+                        bytes = GetByteArray(text.Value);
+                    }
+                    else if (contentAttr.Value == "short_array")
+                    {
+                        bytes = GetUshortArray(text.Value);
+                    }
+                    else
+                    { }
+                }
+                else
+                {
+                    bytes = Encoding.ASCII.GetBytes(text.Value).Concat(new byte[] { 0x00 }).ToArray();
+                }
+                if (bytes != null)
+                {
+                    return new RbfBytes()
+                    {
+                        Name = "",
+                        Value = bytes
+                    };
+                }
             }
 
             return null;
@@ -138,5 +162,45 @@ namespace CodeWalker.GameFiles
                 };
             }
         }
+
+
+
+
+
+        private static byte[] GetByteArray(string text)
+        {
+            if (string.IsNullOrEmpty(text)) return null;
+            var data = new List<byte>();
+            var split = Regex.Split(text, @"[\s\r\n\t]");
+            for (int i = 0; i < split.Length; i++)
+            {
+                if (!string.IsNullOrEmpty(split[i]))
+                {
+                    var str = split[i];
+                    if (string.IsNullOrEmpty(str)) continue;
+                    var val = Convert.ToByte(str);
+                    data.Add(val);
+                }
+            }
+            return data.ToArray();
+        }
+        private static byte[] GetUshortArray(string text)
+        {
+            var data = new List<byte>();
+            var split = Regex.Split(text, @"[\s\r\n\t]");
+            for (int i = 0; i < split.Length; i++)
+            {
+                if (!string.IsNullOrEmpty(split[i]))
+                {
+                    var str = split[i];
+                    if (string.IsNullOrEmpty(str)) continue;
+                    var val = Convert.ToUInt16(str);
+                    data.Add((byte)((val >> 0) & 0xFF));
+                    data.Add((byte)((val >> 8) & 0xFF));
+                }
+            }
+            return data.ToArray();
+        }
+
     }
 }