XML/RBF conversion improvements

This commit is contained in:
dexy 2019-12-30 16:37:29 +11:00
parent e3bbd29b33
commit e40e06896d
4 changed files with 204 additions and 46 deletions

View File

@ -182,6 +182,7 @@ namespace CodeWalker.GameFiles
//TestAudioYmts(); //TestAudioYmts();
//TestMetas(); //TestMetas();
//TestPsos(); //TestPsos();
//TestRbfs();
//TestCuts(); //TestCuts();
//TestYlds(); //TestYlds();
//TestYcds(); //TestYcds();
@ -476,11 +477,11 @@ namespace CodeWalker.GameFiles
//filter ActiveMapFiles based on images.meta? //filter ActiveMapFiles based on images.meta?
//DlcContentDataFile imagesdata; //DlcContentDataFile imagesdata;
//if (imagedatafiles.TryGetValue(path, out imagesdata)) //if (imagedatafiles.TryGetValue(path, out imagesdata))
//{ //{
// ActiveMapRpfFiles[path] = baserpf; // ActiveMapRpfFiles[path] = baserpf;
//} //}
} }
private void InitActiveMapRpfFiles() private void InitActiveMapRpfFiles()
@ -534,9 +535,9 @@ namespace CodeWalker.GameFiles
Dictionary<string, List<string>> overlays = new Dictionary<string, List<string>>(); Dictionary<string, List<string>> overlays = new Dictionary<string, List<string>>();
foreach(var setupfile in DlcSetupFiles) foreach (var setupfile in DlcSetupFiles)
{ {
if(setupfile.DlcFile!=null) if (setupfile.DlcFile != null)
{ {
//if (setupfile.order > maxdlcorder) //if (setupfile.order > maxdlcorder)
// break; // break;
@ -715,7 +716,7 @@ namespace CodeWalker.GameFiles
else else
{ } //how to handle individual files? eg interiorProxies.meta { } //how to handle individual files? eg interiorProxies.meta
} }
private void AddDlcOverlayRpf(string path, string umpath, DlcSetupFile setupfile, Dictionary<string,List<string>> overlays) private void AddDlcOverlayRpf(string path, string umpath, DlcSetupFile setupfile, Dictionary<string, List<string>> overlays)
{ {
string opath = GetDlcOverlayPath(umpath, setupfile); string opath = GetDlcOverlayPath(umpath, setupfile);
if (opath == path) return; if (opath == path) return;
@ -1516,7 +1517,7 @@ namespace CodeWalker.GameFiles
IEnumerable <RpfFile> rpfs = PreloadedMode ? AllRpfs : (IEnumerable<RpfFile>)ActiveMapRpfFiles.Values; IEnumerable<RpfFile> rpfs = PreloadedMode ? AllRpfs : (IEnumerable<RpfFile>)ActiveMapRpfFiles.Values;
var allVehicles = new Dictionary<MetaHash, VehicleInitData>(); var allVehicles = new Dictionary<MetaHash, VehicleInitData>();
@ -1655,7 +1656,7 @@ namespace CodeWalker.GameFiles
return dict; return dict;
} }
var addPedDicts = new Action<string, MetaHash, RpfDirectoryEntry>((namel, hash, dir)=> var addPedDicts = new Action<string, MetaHash, RpfDirectoryEntry>((namel, hash, dir) =>
{ {
Dictionary<MetaHash, RpfFileEntry> dict = null; Dictionary<MetaHash, RpfFileEntry> dict = null;
var files = dir?.Files; var files = dir?.Files;
@ -1963,7 +1964,7 @@ namespace CodeWalker.GameFiles
//ErrorLog("Drawable dictionary not found: " + JenkIndex.GetString(hash)); //too spammy... //ErrorLog("Drawable dictionary not found: " + JenkIndex.GetString(hash)); //too spammy...
} }
} }
else if(!ydd.Loaded) else if (!ydd.Loaded)
{ {
TryLoadEnqueue(ydd); TryLoadEnqueue(ydd);
} }
@ -2236,7 +2237,7 @@ namespace CodeWalker.GameFiles
public bool LoadFile<T>(T file) where T:GameFile,PackedFile public bool LoadFile<T>(T file) where T : GameFile, PackedFile
{ {
if (file == null) return false; if (file == null) return false;
RpfFileEntry entry = file.RpfFileEntry; RpfFileEntry entry = file.RpfFileEntry;
@ -2384,7 +2385,7 @@ namespace CodeWalker.GameFiles
public YtdFile TryGetParentYtd(uint hash) public YtdFile TryGetParentYtd(uint hash)
{ {
MetaHash phash; MetaHash phash;
if(textureParents.TryGetValue(hash, out phash)) if (textureParents.TryGetValue(hash, out phash))
{ {
return GetYtd(phash); return GetYtd(phash);
} }
@ -2890,7 +2891,7 @@ namespace CodeWalker.GameFiles
if (xml.Length != xml2.Length) if (xml.Length != xml2.Length)
{ } { }
if ((xml != xml2)&&(!n.EndsWith("srl.ymt") && !n.StartsWith("des_"))) if ((xml != xml2) && (!n.EndsWith("srl.ymt") && !n.StartsWith("des_")))
{ } { }
} }
@ -3008,6 +3009,87 @@ namespace CodeWalker.GameFiles
{ {
} }
} }
public void TestRbfs()
{
var exceptions = new List<Exception>();
var allrbfs = new List<string>();
var diffrbfs = new List<string>();
foreach (RpfFile file in AllRpfs)
{
foreach (RpfEntry entry in file.AllEntries)
{
var n = entry.NameLower;
if (!(n.EndsWith(".ymt") ||
n.EndsWith(".ymf") ||
n.EndsWith(".ymap") ||
n.EndsWith(".ytyp") ||
n.EndsWith(".cut")))
continue; //PSO files seem to only have these extensions
var fentry = entry as RpfFileEntry;
var data = entry.File.ExtractFile(fentry);
if (data != null)
{
using (MemoryStream ms = new MemoryStream(data))
{
if (RbfFile.IsRBF(ms))
{
UpdateStatus(string.Format(entry.Path));
var rbf = new RbfFile();
rbf.Load(ms);
allrbfs.Add(fentry.Path);
var xml = RbfXml.GetXml(rbf);
if (!string.IsNullOrEmpty(xml))
{ }
var xdoc = new XmlDocument();
xdoc.LoadXml(xml);
var rbf2 = XmlRbf.GetRbf(xdoc);
var rbf2b = rbf2.Save();
var rbf3 = new RbfFile();
rbf3.Load(rbf2b);
var xml3 = RbfXml.GetXml(rbf3);
if (xml.Length != xml3.Length)
{ }
if (xml != xml3)
{
diffrbfs.Add(fentry.Path);
}
if (data.Length != rbf2b.Length)
{
//File.WriteAllBytes("C:\\GitHub\\CodeWalkerResearch\\RBF\\" + fentry.Name + ".dat0", data);
//File.WriteAllBytes("C:\\GitHub\\CodeWalkerResearch\\RBF\\" + fentry.Name + ".dat1", rbf2b);
}
else
{
for (int i = 0; i < data.Length; i++)
{
if (data[i] != rbf2b[i])
{
diffrbfs.Add(fentry.Path);
break;
}
}
}
}
}
}
}
}
string allrbfpaths = string.Join("\r\n", allrbfs);
string diffrbfpaths = string.Join("\r\n", diffrbfs);
}
public void TestCuts() public void TestCuts()
{ {

View File

@ -1553,7 +1553,7 @@ namespace CodeWalker.GameFiles
int cind = indent + 1; int cind = indent + 1;
bool oneline = ((rs.Children.Count == 1) && (rs.Children[0].Name == null)); bool oneline = ((rs.Children.Count == 1) && (rs.Children[0].Name == null) && (rs.Attributes.Count == 0));
OpenTag(sb, indent, rs.Name + attStr, !oneline); OpenTag(sb, indent, rs.Name + attStr, !oneline);
@ -1563,17 +1563,14 @@ namespace CodeWalker.GameFiles
if (child is RbfBytes) if (child is RbfBytes)
{ {
var bytesChild = (RbfBytes)child; var bytesChild = (RbfBytes)child;
var contentField = rs.FindChild("content") as RbfString;//TODO: fix this to output nicer XML! var contentField = rs.FindAttribute("content") as RbfString;//TODO: fix this to output nicer XML!
if (contentField != null) if (contentField != null)
{ {
OpenTag(sb, cind, "value");
var aind = cind + 1;
if (contentField.Value == "char_array") if (contentField.Value == "char_array")
{ {
foreach (byte k in bytesChild.Value) foreach (byte k in bytesChild.Value)
{ {
Indent(sb, aind); Indent(sb, cind);
sb.AppendLine(k.ToString()); sb.AppendLine(k.ToString());
} }
} }
@ -1582,23 +1579,20 @@ namespace CodeWalker.GameFiles
var valueReader = new DataReader(new MemoryStream(bytesChild.Value)); var valueReader = new DataReader(new MemoryStream(bytesChild.Value));
while (valueReader.Position < valueReader.Length) while (valueReader.Position < valueReader.Length)
{ {
Indent(sb, aind); Indent(sb, cind);
var y = valueReader.ReadUInt16(); var y = valueReader.ReadUInt16();
sb.AppendLine(y.ToString()); sb.AppendLine(y.ToString());
} }
} }
else else
{ {
ErrorXml(sb, aind, "Unexpected content type: " + contentField.Value); ErrorXml(sb, cind, "Unexpected content type: " + contentField.Value);
} }
CloseTag(sb, cind, "value");
} }
else else
{ {
string stringValue = Encoding.ASCII.GetString(bytesChild.Value); string stringValue = Encoding.ASCII.GetString(bytesChild.Value);
string str = stringValue.Substring(0, stringValue.Length - 1); //removes null terminator string str = stringValue.Substring(0, stringValue.Length - 1); //removes null terminator
sb.Append(str); sb.Append(str);
} }
} }
@ -1609,6 +1603,8 @@ namespace CodeWalker.GameFiles
} }
if (child is RbfString) if (child is RbfString)
{ {
////// this doesn't seem to be used! it's always using RbfBytes child...
var stringChild = (RbfString)child; var stringChild = (RbfString)child;
StringTag(sb, cind, stringChild.Name, stringChild.Value); StringTag(sb, cind, stringChild.Name, stringChild.Value);
@ -1940,14 +1936,14 @@ namespace CodeWalker.GameFiles
public static string UintString(uint h) public static string UintString(uint h)
{ {
string str; //string str;
if (MetaNames.TryGetString(h, out str)) return str; //if (MetaNames.TryGetString(h, out str)) return str;
str = JenkIndex.TryGetString(h); //str = JenkIndex.TryGetString(h);
if (!string.IsNullOrEmpty(str)) return str; //if (!string.IsNullOrEmpty(str)) return str;
//TODO: do extra hash lookup here ////TODO: do extra hash lookup here
//if(Lookup.TryGetValue(uh, out str)) ... ////if(Lookup.TryGetValue(uh, out str)) ...
//if (h == 0) return ""; //if (h == 0) return "";

View File

@ -44,6 +44,13 @@ namespace CodeWalker.GameFiles
public List<RbfEntryDescription> descriptors { get; set; } public List<RbfEntryDescription> descriptors { get; set; }
public Dictionary<string, int> outDescriptors { get; private set; } = new Dictionary<string, int>(); public Dictionary<string, int> outDescriptors { get; private set; } = new Dictionary<string, int>();
public void Load(byte[] data)
{
using (var ms = new MemoryStream(data))
Load(ms);
}
public RbfStructure Load(string fileName) public RbfStructure Load(string fileName)
{ {
using (var fileStream = new FileStream(fileName, FileMode.Open)) using (var fileStream = new FileStream(fileName, FileMode.Open))
@ -222,7 +229,7 @@ namespace CodeWalker.GameFiles
public byte GetDescriptorIndex(IRbfType t, out bool isNew) public byte GetDescriptorIndex(IRbfType t, out bool isNew)
{ {
var key = $"{t.Name}_{t.DataType}"; var key = t.Name;// $"{t.Name}_{t.DataType}";
isNew = false; isNew = false;
if (!outDescriptors.TryGetValue(key, out var idx)) if (!outDescriptors.TryGetValue(key, out var idx))
@ -388,6 +395,15 @@ namespace CodeWalker.GameFiles
} }
return null; return null;
} }
public IRbfType FindAttribute(string name)
{
foreach (var attr in Attributes)
{
if (attr == null) continue;
if (attr.Name == name) return attr;
}
return null;
}
public void Save(RbfFile root, DataWriter writer) public void Save(RbfFile root, DataWriter writer)
{ {
root.WriteRecordId(this, writer); root.WriteRecordId(this, writer);

View File

@ -56,13 +56,15 @@ namespace CodeWalker.GameFiles
Z = z Z = z
}; };
} }
else if ((element.Elements().Count() == 0) && (element.Attributes().Count() == 0)) //else if (element.Name == "type" || element.Name == "key" || element.Name == "platform") else if ((element.Elements().Count() == 0) && (element.Attributes().Count() == 0) && (!element.IsEmpty)) //else if (element.Name == "type" || element.Name == "key" || element.Name == "platform")
{ {
return new RbfString() var bytearr = Encoding.ASCII.GetBytes(element.Value);
{ var bytearrnt = new byte[bytearr.Length + 1];
Name = element.Name.LocalName, Buffer.BlockCopy(bytearr, 0, bytearrnt, 0, bytearr.Length);
Value = element.Value var bytes = new RbfBytes() { Value = bytearrnt };
}; var struc = new RbfStructure() { Name = element.Name.LocalName };
struc.Children.Add(bytes);
return struc;
} }
var n = new RbfStructure(); var n = new RbfStructure();
@ -83,11 +85,33 @@ namespace CodeWalker.GameFiles
} }
else if (node is XText text) else if (node is XText text)
{ {
return new RbfBytes() byte[] bytes = null;
var contentAttr = node.Parent?.Attribute("content");
if (contentAttr != null)
{ {
Name = "", if (contentAttr.Value == "char_array")
Value = Encoding.ASCII.GetBytes(text.Value).Concat(new byte[] { 0x00 }).ToArray() {
}; bytes = GetByteArray(text.Value);
}
else if (contentAttr.Value == "short_array")
{
bytes = GetUshortArray(text.Value);
}
else
{ }
}
else
{
bytes = Encoding.ASCII.GetBytes(text.Value).Concat(new byte[] { 0x00 }).ToArray();
}
if (bytes != null)
{
return new RbfBytes()
{
Name = "",
Value = bytes
};
}
} }
return null; return null;
@ -138,5 +162,45 @@ namespace CodeWalker.GameFiles
}; };
} }
} }
private static byte[] GetByteArray(string text)
{
if (string.IsNullOrEmpty(text)) return null;
var data = new List<byte>();
var split = Regex.Split(text, @"[\s\r\n\t]");
for (int i = 0; i < split.Length; i++)
{
if (!string.IsNullOrEmpty(split[i]))
{
var str = split[i];
if (string.IsNullOrEmpty(str)) continue;
var val = Convert.ToByte(str);
data.Add(val);
}
}
return data.ToArray();
}
private static byte[] GetUshortArray(string text)
{
var data = new List<byte>();
var split = Regex.Split(text, @"[\s\r\n\t]");
for (int i = 0; i < split.Length; i++)
{
if (!string.IsNullOrEmpty(split[i]))
{
var str = split[i];
if (string.IsNullOrEmpty(str)) continue;
var val = Convert.ToUInt16(str);
data.Add((byte)((val >> 0) & 0xFF));
data.Add((byte)((val >> 8) & 0xFF));
}
}
return data.ToArray();
}
} }
} }