Heightmap/XML conversion

CodeWalker.Core/GameFiles/FileTypes/HeightmapFile.cs

@@ -0,0 +1,342 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using TC = System.ComponentModel.TypeConverterAttribute;
+using EXP = System.ComponentModel.ExpandableObjectConverter;
+using SharpDX;
+using System.IO;
+using System.Xml;
+
+namespace CodeWalker.GameFiles
+{
+    [TC(typeof(EXP))]
+    public class HeightmapFile : GameFile, PackedFile
+    {
+        public byte[] RawFileData { get; set; }
+
+        public uint Magic { get; set; } = 0x484D4150; //'HMAP'
+        public byte VersionMajor { get; set; } = 1;
+        public byte VersionMinor { get; set; } = 1;
+        public ushort Pad { get; set; }
+        public uint Compressed { get; set; } = 1;
+        public ushort Width { get; set; }
+        public ushort Height { get; set; }
+        public Vector3 BBMin { get; set; }
+        public Vector3 BBMax { get; set; }
+        public uint Length { get; set; }
+        public CompHeader[] CompHeaders { get; set; }
+        public byte[] MaxHeights { get; set; }
+        public byte[] MinHeights { get; set; }
+
+        public HeightmapFile() : base(null, GameFileType.Heightmap)
+        {
+        }
+        public HeightmapFile(RpfFileEntry entry) : base(entry, GameFileType.Heightmap)
+        {
+            RpfFileEntry = entry;
+        }
+
+        public void Load(byte[] data, RpfFileEntry entry)
+        {
+            RawFileData = data;
+            if (entry != null)
+            {
+                RpfFileEntry = entry;
+                Name = entry.Name;
+            }
+
+            using (MemoryStream ms = new MemoryStream(data))
+            {
+                DataReader r = new DataReader(ms, Endianess.BigEndian);
+
+                Read(r);
+            }
+        }
+
+        public byte[] Save()
+        {
+            MemoryStream s = new MemoryStream();
+            DataWriter w = new DataWriter(s, Endianess.BigEndian);
+
+            Write(w);
+
+            var buf = new byte[s.Length];
+            s.Position = 0;
+            s.Read(buf, 0, buf.Length);
+            return buf;
+        }
+
+        private void Read(DataReader r)
+        {
+            Magic = r.ReadUInt32();
+            VersionMajor = r.ReadByte();
+            VersionMinor = r.ReadByte();
+            Pad = r.ReadUInt16();
+            Compressed = r.ReadUInt32();
+            Width = r.ReadUInt16();
+            Height = r.ReadUInt16();
+            BBMin = r.ReadVector3();
+            BBMax = r.ReadVector3();
+            Length = r.ReadUInt32();
+
+
+            if (Length != (r.Length - r.Position))
+            { }
+
+
+            var dlen = (int)Length;
+            if (Compressed > 0)
+            {
+                CompHeaders = new CompHeader[Height];
+                for (int i = 0; i < Height; i++)
+                {
+                    CompHeaders[i].Read(r);
+                }
+                dlen -= (Height * 8);
+            }
+
+            if ((r.Length - r.Position) != dlen)
+            { }
+
+            var d = r.ReadBytes(dlen);
+
+            if ((r.Length - r.Position) != 0)
+            { }
+
+            if (Compressed > 0)
+            {
+                MaxHeights = new byte[Width * Height];
+                MinHeights = new byte[Width * Height];
+                var h2off = dlen / 2; //is this right?
+                for (int y = 0; y < Height; y++)
+                {
+                    var h = CompHeaders[y];
+                    for (int i = 0; i < h.Count; i++)
+                    {
+                        int x = h.Start + i;
+                        int o = h.DataOffset + x;
+                        MaxHeights[y * Width + x] = d[o];
+                        MinHeights[y * Width + x] = d[o + h2off];
+                    }
+                    for (int x = 0; x < Width; x++)
+                    {
+                        var hm1v = MaxHeights[y * Width + x];
+                        var hm2v = MinHeights[y * Width + x];
+                        var diff = hm1v - hm2v;
+                        if ((diff <= 0) && (hm1v != 0))
+                        { }
+                    }
+                }
+            }
+            else
+            {
+                MaxHeights = d; //no way to test this as vanilla heightmaps are compressed...
+                MinHeights = d; //this won't work anyway.
+            }
+
+        }
+        private void Write(DataWriter w)
+        {
+            var d = MaxHeights;
+            if (Compressed > 0)
+            {
+                var ch = new CompHeader[Height];
+                var d1 = new List<byte>();
+                var d2 = new List<byte>();
+                for (int y = 0; y < Height; y++)
+                {
+                    var start = 0;
+                    var end = 0;
+                    for (int x = 0; x < Width; x++)
+                    {
+                        if (MaxHeights[y * Width + x] != 0) { start = x; break; }
+                    }
+                    for (int x = Width - 1; x >= 0; x--)
+                    {
+                        if (MaxHeights[y * Width + x] != 0) { end = x + 1; break; }
+                    }
+                    var count = end - start;
+                    var offset = (count > 0) ? d1.Count - start : 0;
+                    for (int i = 0; i < count; i++)
+                    {
+                        var x = start + i;
+                        var n = y * Width + x;
+                        d1.Add(MaxHeights[n]);
+                        d2.Add(MinHeights[n]);
+                    }
+                    var h = new CompHeader() { Start = (ushort)start, Count = (ushort)count, DataOffset = offset };
+                    ch[y] = h;
+                }
+                d1.AddRange(d2);//the 2 sets of compressed data are just smushed together
+                d = d1.ToArray();
+                CompHeaders = ch;
+                Length = (uint)(d.Length + Height * 8);
+            }
+            else
+            {
+                Length = (uint)d.Length;
+            }
+
+
+            w.Write(Magic);
+            w.Write(VersionMajor);
+            w.Write(VersionMinor);
+            w.Write(Pad);
+            w.Write(Compressed);
+            w.Write(Width);
+            w.Write(Height);
+            w.Write(BBMin);
+            w.Write(BBMax);
+            w.Write(Length);
+            if (Compressed > 0)
+            {
+                for (int i = 0; i < Height; i++)
+                {
+                    CompHeaders[i].Write(w);
+                }
+            }
+            w.Write(d);
+        }
+
+
+        public void WriteXml(StringBuilder sb, int indent)
+        {
+            HmapXml.ValueTag(sb, indent, "Width", Width.ToString());
+            HmapXml.ValueTag(sb, indent, "Height", Height.ToString());
+            HmapXml.SelfClosingTag(sb, indent, "BBMin " + FloatUtil.GetVector3XmlString(BBMin));
+            HmapXml.SelfClosingTag(sb, indent, "BBMax " + FloatUtil.GetVector3XmlString(BBMax));
+            HmapXml.WriteRawArray(sb, InvertImage(MaxHeights, Width, Height), indent, "MaxHeights", "", HmapXml.FormatHexByte, Width);
+            HmapXml.WriteRawArray(sb, InvertImage(MinHeights, Width, Height), indent, "MinHeights", "", HmapXml.FormatHexByte, Width);
+        }
+        public void ReadXml(XmlNode node)
+        {
+            Width = (ushort)Xml.GetChildUIntAttribute(node, "Width");
+            Height = (ushort)Xml.GetChildUIntAttribute(node, "Height");
+            BBMin = Xml.GetChildVector3Attributes(node, "BBMin");
+            BBMax = Xml.GetChildVector3Attributes(node, "BBMax");
+            MaxHeights = InvertImage(Xml.GetChildRawByteArray(node, "MaxHeights"), Width, Height);
+            MinHeights = InvertImage(Xml.GetChildRawByteArray(node, "MinHeights"), Width, Height);
+        }
+
+
+
+
+
+
+        private byte[] InvertImage(byte[] i, int w, int h)
+        {
+            //inverts the image vertically
+            byte[] o = new byte[i.Length];
+            for (int y = 0; y < h; y++)
+            {
+                int io = y * w;
+                int oo = (h - y - 1) * w;
+                Buffer.BlockCopy(i, io, o, oo, w);
+            }
+            return o;
+        }
+
+
+
+
+
+        public string GetPGM()
+        {
+            if (MaxHeights == null) return string.Empty;
+
+            var sb = new StringBuilder();
+            sb.AppendFormat("P2\n{0} {1}\n255\n", Width, Height);
+
+            for (int y = Height - 1; y >= 0; y--)
+            {
+                for (int x = 0; x < Width; x++)
+                {
+                    var h = MaxHeights[y * Width + x];
+                    sb.Append(h.ToString());
+                    sb.Append(" ");
+                }
+                sb.Append("\n");
+            }
+
+            return sb.ToString();
+        }
+
+
+
+        public struct CompHeader
+        {
+            public ushort Start;
+            public ushort Count;
+            public int DataOffset;
+
+            public void Read(DataReader r)
+            {
+                Start = r.ReadUInt16();
+                Count = r.ReadUInt16();
+                DataOffset = r.ReadInt32();
+            }
+            public void Write(DataWriter w)
+            {
+                w.Write(Start);
+                w.Write(Count);
+                w.Write(DataOffset);
+            }
+
+            public override string ToString()
+            {
+                return Start.ToString() + ", " + Count.ToString() + ", " + DataOffset.ToString();
+            }
+        }
+
+    }
+
+
+    public class HmapXml : MetaXmlBase
+    {
+
+        public static string GetXml(HeightmapFile hmf)
+        {
+            StringBuilder sb = new StringBuilder();
+            sb.AppendLine(XmlHeader);
+
+            if ((hmf != null) && (hmf.MaxHeights != null))
+            {
+                var name = "Heightmap";
+
+                OpenTag(sb, 0, name);
+
+                hmf.WriteXml(sb, 1);
+
+                CloseTag(sb, 0, name);
+            }
+
+            return sb.ToString();
+        }
+
+
+    }
+
+
+    public class XmlHmap
+    {
+
+        public static HeightmapFile GetHeightmap(string xml)
+        {
+            XmlDocument doc = new XmlDocument();
+            doc.LoadXml(xml);
+            return GetHeightmap(doc);
+        }
+
+        public static HeightmapFile GetHeightmap(XmlDocument doc)
+        {
+            HeightmapFile hmf = new HeightmapFile();
+            hmf.ReadXml(doc.DocumentElement);
+            return hmf;
+        }
+
+
+    }
+
+
+
+}

CodeWalker.Core/GameFiles/GameFile.cs

@@ -80,6 +80,7 @@ namespace CodeWalker.GameFiles
         Yed = 24,
         Yld = 25,
         Yfd = 26,
+        Heightmap = 27,
     }
 
 

CodeWalker.Core/GameFiles/GameFileCache.cs

@@ -199,6 +199,7 @@ namespace CodeWalker.GameFiles
                 //TestYmaps();
                 //TestPlacements();
                 //TestDrawables();
+                //TestHeightmaps();
                 //GetShadersXml();
                 //GetArchetypeTimesList();
                 //string typestr = PsoTypes.GetTypesString();
@@ -4502,6 +4503,41 @@ namespace CodeWalker.GameFiles
             UpdateStatus((DateTime.Now - starttime).ToString() + " elapsed, " + drawablecount.ToString() + " drawables, " + errs.Count.ToString() + " errors.");
 
         }
+        public void TestHeightmaps()
+        {
+            var errorfiles = new List<RpfEntry>();
+            foreach (RpfFile file in AllRpfs)
+            {
+                foreach (RpfEntry entry in file.AllEntries)
+                {
+                    if (entry.NameLower.EndsWith(".dat") && entry.NameLower.StartsWith("heightmap"))
+                    {
+                        UpdateStatus(string.Format(entry.Path));
+                        HeightmapFile hmf = null;
+                        hmf = RpfMan.GetFile<HeightmapFile>(entry);
+                        var d1 = hmf.RawFileData;
+                        //var d2 = hmf.Save();
+                        var xml = HmapXml.GetXml(hmf);
+                        var hmf2 = XmlHmap.GetHeightmap(xml);
+                        var d2 = hmf2.Save();
+
+                        if (d1.Length == d2.Length)
+                        {
+                            for (int i = 0; i < d1.Length; i++)
+                            {
+                                if (d1[i] != d2[i])
+                                { }
+                            }
+                        }
+                        else
+                        { }
+
+                    }
+                }
+            }
+            if (errorfiles.Count > 0)
+            { }
+        }
         public void GetShadersXml()
         {
             bool doydr = true;

CodeWalker.Core/GameFiles/MetaTypes/MetaXml.cs

@@ -112,6 +112,16 @@ namespace CodeWalker.GameFiles
                 AwcFile awc = RpfFile.GetFile<AwcFile>(e, data);
                 return GetXml(awc, out filename, outputfolder);
             }
+            else if (fnl.EndsWith("cache_y.dat"))
+            {
+                CacheDatFile cdf = RpfFile.GetFile<CacheDatFile>(e, data);
+                return GetXml(cdf, out filename, outputfolder);
+            }
+            else if (fnl.EndsWith(".dat") && fnl.StartsWith("heightmap"))
+            {
+                HeightmapFile hmf = RpfFile.GetFile<HeightmapFile>(e, data);
+                return GetXml(hmf, out filename, outputfolder);
+            }
             filename = fn;
             return string.Empty;
         }
@@ -237,6 +247,18 @@ namespace CodeWalker.GameFiles
             filename = fn + ".xml";
             return AwcXml.GetXml(awc, outputfolder);
         }
+        public static string GetXml(CacheDatFile cdf, out string filename, string outputfolder)
+        {
+            var fn = (cdf?.FileEntry?.Name) ?? "";
+            filename = fn + ".xml";
+            return cdf.GetXml();
+        }
+        public static string GetXml(HeightmapFile hmf, out string filename, string outputfolder)
+        {
+            var fn = (hmf?.Name) ?? "";
+            filename = fn + ".xml";
+            return HmapXml.GetXml(hmf);
+        }
 
 
 
@@ -2148,6 +2170,7 @@ namespace CodeWalker.GameFiles
         Ypt = 14,
         Yld = 15,
         Awc = 16,
+        Heightmap = 17,
     }
 
 }

CodeWalker/ExploreForm.cs

@@ -308,19 +308,21 @@ namespace CodeWalker
             InitFileType(".awc", "Audio Wave Container", 22, FileTypeAction.ViewAwc, true);
             InitFileType(".rel", "Audio Data (REL)", 23, FileTypeAction.ViewRel, true);
 
-            InitSubFileType(".dat", "cache_y.dat", "Cache File", 6, FileTypeAction.ViewCacheDat);
+            InitSubFileType(".dat", "cache_y.dat", "Cache File", 6, FileTypeAction.ViewCacheDat, true);
+            InitSubFileType(".dat", "heightmap.dat", "Heightmap", 6, FileTypeAction.ViewHeightmap, true);
+            InitSubFileType(".dat", "heightmapheistisland.dat", "Heightmap", 6, FileTypeAction.ViewHeightmap, true);
         }
         private void InitFileType(string ext, string name, int imgidx, FileTypeAction defaultAction = FileTypeAction.ViewHex, bool xmlConvertible = false)
         {
             var ft = new FileTypeInfo(ext, name, imgidx, defaultAction, xmlConvertible);
             FileTypes[ext] = ft;
         }
-        private void InitSubFileType(string ext, string subext, string name, int imgidx, FileTypeAction defaultAction = FileTypeAction.ViewHex)
+        private void InitSubFileType(string ext, string subext, string name, int imgidx, FileTypeAction defaultAction = FileTypeAction.ViewHex, bool xmlConvertible = false)
         {
             FileTypeInfo pti = null;
             if (FileTypes.TryGetValue(ext, out pti))
             {
-                var ft = new FileTypeInfo(subext, name, imgidx, defaultAction, pti.XmlConvertible);
+                var ft = new FileTypeInfo(subext, name, imgidx, defaultAction, xmlConvertible);
                 pti.AddSubType(ft);
             }
         }
@@ -1398,6 +1400,7 @@ namespace CodeWalker
                 case FileTypeAction.ViewYed:
                 case FileTypeAction.ViewYld:
                 case FileTypeAction.ViewYfd:
+                case FileTypeAction.ViewHeightmap:
                     return true;
                 case FileTypeAction.ViewHex:
                 default:
@@ -1524,6 +1527,9 @@ namespace CodeWalker
                     case FileTypeAction.ViewYfd:
                         ViewYfd(name, path, data, fe);
                         break;
+                    case FileTypeAction.ViewHeightmap:
+                        ViewHeightmap(name, path, data, fe);
+                        break;
                     case FileTypeAction.ViewHex:
                     default:
                         ViewHex(name, path, data);
@@ -1758,6 +1764,13 @@ namespace CodeWalker
             f.Show();
             f.LoadMeta(cachedat);
         }
+        private void ViewHeightmap(string name, string path, byte[] data, RpfFileEntry e)
+        {
+            var heightmap = RpfFile.GetFile<HeightmapFile>(e, data);
+            MetaForm f = new MetaForm(this);
+            f.Show();
+            f.LoadMeta(heightmap);
+        }
 
         private RpfFileEntry CreateFileEntry(string name, string path, ref byte[] data)
         {
@@ -2643,6 +2656,14 @@ namespace CodeWalker
                     {
                         mformat = MetaFormat.Awc;
                     }
+                    if (fnamel.EndsWith("cache_y.dat.xml"))
+                    {
+                        mformat = MetaFormat.CacheFile;
+                    }
+                    if (fnamel.EndsWith(".dat.xml") && fnamel.StartsWith("heightmap"))
+                    {
+                        mformat = MetaFormat.Heightmap;
+                    }
 
                     fname = fname.Substring(0, fname.Length - trimlength);
                     fnamel = fnamel.Substring(0, fnamel.Length - trimlength);
@@ -2825,6 +2846,24 @@ namespace CodeWalker
                                 data = awc.Save();
                                 break;
                             }
+                        case MetaFormat.CacheFile:
+                            {
+                                var cdf = new CacheDatFile();
+                                //cdf.LoadXml() //TODO!!!
+                                MessageBox.Show(fname + ": CacheFile XML import still TODO!!!", "Cannot import CacheFile XML");
+                                break;
+                            }
+                        case MetaFormat.Heightmap:
+                            {
+                                var hmf = XmlHmap.GetHeightmap(doc);
+                                if (hmf.MaxHeights == null)
+                                {
+                                    MessageBox.Show(fname + ": Schema not supported.", "Cannot import Heightmap XML");
+                                    continue;
+                                }
+                                data = hmf.Save();
+                                break;
+                            }
                     }
 
 
@@ -4653,6 +4692,7 @@ namespace CodeWalker
         ViewYed = 20,
         ViewYld = 21,
         ViewYfd = 22,
+        ViewHeightmap = 23,
     }
 
 

CodeWalker/Forms/MetaForm.cs

@@ -332,6 +332,20 @@ namespace CodeWalker.Forms
                 metaFormat = MetaFormat.CacheFile;
             }
         }
+        public void LoadMeta(HeightmapFile heightmap)
+        {
+            var fn = ((heightmap?.RpfFileEntry?.Name) ?? "") + ".xml";
+            Xml = HmapXml.GetXml(heightmap);
+            FileName = fn;
+            RawPropertyGrid.SelectedObject = heightmap;
+            rpfFileEntry = heightmap?.RpfFileEntry;
+            modified = false;
+            metaFormat = MetaFormat.XML;
+            if (heightmap?.RpfFileEntry != null)
+            {
+                metaFormat = MetaFormat.Heightmap;
+            }
+        }
 
 
 
@@ -393,6 +407,15 @@ namespace CodeWalker.Forms
                     case MetaFormat.CacheFile:
                         MessageBox.Show("Sorry, CacheFile import is not supported.", "Cannot import CacheFile XML");
                         return false;
+                    case MetaFormat.Heightmap:
+                        var hmap = XmlHmap.GetHeightmap(doc);
+                        if (hmap.MaxHeights == null)
+                        {
+                            MessageBox.Show("Schema not supported.", "Cannot import Heightmap XML");
+                            return false;
+                        }
+                        data = hmap.Save();
+                        break;
                 }
             }
 #if !DEBUG