RPF Explorer archive defragmenting tool

ExploreForm.Designer.cs

@@ -133,6 +133,8 @@
             this.EditModeModsWarningPanel = new System.Windows.Forms.Panel();
             this.pictureBox2 = new System.Windows.Forms.PictureBox();
             this.label2 = new System.Windows.Forms.Label();
+            this.ListContextDefragmentMenu = new System.Windows.Forms.ToolStripMenuItem();
+            this.ListContextDefragmentSeparator = new System.Windows.Forms.ToolStripSeparator();
             this.MainMenu.SuspendLayout();
             this.MainToolbar.SuspendLayout();
             this.StatusBar.SuspendLayout();
@@ -173,7 +175,7 @@
             // 
             this.FileExitMenu.Name = "FileExitMenu";
             this.FileExitMenu.ShortcutKeys = ((System.Windows.Forms.Keys)((System.Windows.Forms.Keys.Alt | System.Windows.Forms.Keys.F4)));
-            this.FileExitMenu.Size = new System.Drawing.Size(134, 22);
+            this.FileExitMenu.Size = new System.Drawing.Size(152, 22);
             this.FileExitMenu.Text = "Exit";
             this.FileExitMenu.Click += new System.EventHandler(this.FileExitMenu_Click);
             // 
@@ -777,9 +779,11 @@
             this.ListContextReplaceMenu,
             this.ListContextDeleteMenu,
             this.ListContextEditSeparator,
+            this.ListContextDefragmentMenu,
+            this.ListContextDefragmentSeparator,
             this.ListContextSelectAllMenu});
             this.ListContextMenu.Name = "MainContextMenu";
-            this.ListContextMenu.Size = new System.Drawing.Size(208, 414);
+            this.ListContextMenu.Size = new System.Drawing.Size(208, 464);
             // 
             // ListContextViewMenu
             // 
@@ -1095,6 +1099,18 @@
             this.label2.TabIndex = 0;
             this.label2.Text = "You are editing files in the mods folder";
             // 
+            // ListContextDefragmentMenu
+            // 
+            this.ListContextDefragmentMenu.Name = "ListContextDefragmentMenu";
+            this.ListContextDefragmentMenu.Size = new System.Drawing.Size(207, 22);
+            this.ListContextDefragmentMenu.Text = "Defragment Archive...";
+            this.ListContextDefragmentMenu.Click += new System.EventHandler(this.ListContextDefragmentMenu_Click);
+            // 
+            // ListContextDefragmentSeparator
+            // 
+            this.ListContextDefragmentSeparator.Name = "ListContextDefragmentSeparator";
+            this.ListContextDefragmentSeparator.Size = new System.Drawing.Size(204, 6);
+            // 
             // ExploreForm
             // 
             this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
@@ -1237,5 +1253,7 @@
         private System.Windows.Forms.Panel EditModeModsWarningPanel;
         private System.Windows.Forms.PictureBox pictureBox2;
         private System.Windows.Forms.Label label2;
+        private System.Windows.Forms.ToolStripMenuItem ListContextDefragmentMenu;
+        private System.Windows.Forms.ToolStripSeparator ListContextDefragmentSeparator;
     }
 }

ExploreForm.cs

@@ -1536,6 +1536,7 @@ namespace CodeWalker
             bool canimport = EditMode && (CurrentFolder?.RpfFolder != null) && !issearch;
             bool cancreate = EditMode && !issearch;
             bool canedit = false;
+            bool candefrag = false;
 
             if (item != null)
             {
@@ -1549,6 +1550,7 @@ namespace CodeWalker
                 canexportxml = CanExportXml(item);
                 canedit = EditMode && !issearch;
                 canextract = isfile || (isarchive && !isfilesys);
+                candefrag = isarchive && canedit;
             }
 
 
@@ -1575,6 +1577,9 @@ namespace CodeWalker
             ListContextDeleteMenu.Visible = canedit;
             ListContextEditSeparator.Visible = canedit;
 
+            ListContextDefragmentMenu.Visible = candefrag;
+            ListContextDefragmentSeparator.Visible = candefrag;
+
             ListContextMenu.Show(Cursor.Position);
 
         }
@@ -1606,7 +1611,9 @@ namespace CodeWalker
 
         private void EnsureEditModeWarning()
         {
-            bool show = EditMode && !CurrentFolder.Path.ToLowerInvariant().StartsWith("mods");
+            bool mods = CurrentFolder.Path.ToLowerInvariant().StartsWith("mods");
+            bool srch = CurrentFolder?.IsSearchResults ?? false;
+            bool show = EditMode && !mods && !srch;
             int gap = 3;
             int bot = MainListView.Bottom;
 
@@ -1643,6 +1650,10 @@ namespace CodeWalker
 
             var rpf = CurrentFolder.RpfFolder.File;
 
+            return EnsureRpfValidEncryption(rpf);
+        }
+        private bool EnsureRpfValidEncryption(RpfFile rpf)
+        {
             if (rpf == null) return false;
 
             bool needsupd = false;
@@ -1679,7 +1690,6 @@ namespace CodeWalker
 
 
 
-
         private void ViewSelected()
         {
             for (int i = 0; i < MainListView.SelectedIndices.Count; i++)
@@ -2360,6 +2370,7 @@ namespace CodeWalker
         private void RenameSelected()
         {
             if (!EditMode) return;
+            if (CurrentFolder?.IsSearchResults ?? false) return;
             if (MainListView.SelectedIndices.Count != 1) return;
             var idx = MainListView.SelectedIndices[0];
             if ((CurrentFiles != null) && (CurrentFiles.Count > idx))
@@ -2480,6 +2491,15 @@ namespace CodeWalker
         {
             try
             {
+                if (item.Folder?.RpfFile != null)
+                {
+                    //confirm deletion of RPF archives, just to be friendly.
+                    if (MessageBox.Show("Are you sure you want to delete this archive?\n" + item.Path, "Confirm delete archive", MessageBoxButtons.YesNo) != DialogResult.Yes)
+                    {
+                        return;
+                    }
+                }
+
                 var parent = item.Parent;
                 if (parent.RpfFolder != null)
                 {
@@ -2518,6 +2538,86 @@ namespace CodeWalker
                 return;
             }
         }
+        private void DefragmentSelected()
+        {
+            if (!EditMode) return;
+            if (CurrentFolder?.IsSearchResults ?? false) return;
+            if (MainListView.SelectedIndices.Count != 1)
+            {
+                MessageBox.Show("Can only defragment one item at a time. Please have only one item selected.");
+                return;
+            }
+            var idx = MainListView.SelectedIndices[0];
+            if ((CurrentFiles == null) || (CurrentFiles.Count <= idx))
+            {
+                return;
+            }
+
+            var item = CurrentFiles[idx];
+            var rpf = item.Folder?.RpfFile;
+            if (rpf == null)
+            {
+                MessageBox.Show("Can only defragment RPF archives!");
+                return;
+            }
+
+            Form form = new Form() {
+                Width = 450,
+                Height = 250,
+                FormBorderStyle = FormBorderStyle.FixedDialog,
+                Text = "Defragment RPF Archive - CodeWalker by dexyfex",
+                StartPosition = FormStartPosition.CenterParent,
+                MaximizeBox = false,
+                MinimizeBox = false
+            };
+            var addCtrl = new Func<Control, Control>(c => { form.Controls.Add(c); return c; });
+            var addLabel = new Func<int, string, Control>((y, t) => {
+                return addCtrl(new Label() { Left = 30, Top = y, Width = 370, Height = 20, Text = t });
+            });
+            var rpfNameLabel = addLabel(20, "Archive: " + rpf.Path);
+            var curSizeLabel = addLabel(40, string.Empty);
+            var newSizeLabel = addLabel(60, string.Empty);
+            var redSizeLabel = addLabel(80, string.Empty);
+            var statusLabel = addLabel(110, string.Empty);
+            var progressBar = addCtrl(new ProgressBar() { Left = 30, Top = 130, Width = 370, Height = 20, Minimum = 0, Maximum = 1000, MarqueeAnimationSpeed = 50 }) as ProgressBar;
+            var beginButton = addCtrl(new Button() { Text = "Begin Defragment", Left = 30, Top = 170, Width = 120 }) as Button;
+            var closeButton = addCtrl(new Button() { Text = "Close", Left = 320, Top = 170, Width = 80 }) as Button;
+            var inProgress = false;
+            var updateProgress = new Action<string, float>((s, p) => { form.Invoke(new Action(() => {
+                statusLabel.Text = s;
+                progressBar.Value = Math.Max(0, Math.Min((int)(p * 1000), 1000));//p in range 0..1
+            }));});
+            var updateSizeLabels = new Action<bool>((init) => {
+                var curSize = rpf.FileSize;
+                var newSize = rpf.GetDefragmentedFileSize();
+                var redSize = curSize - newSize;
+                curSizeLabel.Text = "Archive current size: " + TextUtil.GetBytesReadable(curSize);
+                newSizeLabel.Text = "Defragmented size: " + TextUtil.GetBytesReadable(newSize);
+                redSizeLabel.Text = "Size reduction: " + TextUtil.GetBytesReadable(redSize);
+                if (init) statusLabel.Text = (redSize <= 0) ? "Defragmentation not required." : "Ready to defragment.";
+            });
+            var enableUi = new Action<bool>(enable => { form.Invoke(new Action(() => {
+                beginButton.Enabled = enable;
+                closeButton.Enabled = enable;
+            }));});
+            var defragment = new Action(() => { Task.Run(() => {
+                if (inProgress) return;
+                if (!EnsureRpfValidEncryption(rpf)) return;
+                inProgress = true;
+                enableUi(false);
+                RpfFile.Defragment(rpf, updateProgress);
+                updateProgress("Defragment complete.", 1.0f);
+                enableUi(true);
+                form.Invoke(new Action(() => { updateSizeLabels(false); }));
+                inProgress = false;
+            });});
+            updateSizeLabels(true);
+            beginButton.Click += (sender, e) => { defragment(); };
+            closeButton.Click += (sender, e) => { form.Close(); };
+            form.FormClosing += (s, e) => { e.Cancel = inProgress; };
+            form.ShowDialog(this);
+            RefreshMainListView();
+        }
         private void SelectAll()
         {
             MainListView.SelectAllItems();
@@ -3007,6 +3107,11 @@ namespace CodeWalker
             DeleteSelected();
         }
 
+        private void ListContextDefragmentMenu_Click(object sender, EventArgs e)
+        {
+            DefragmentSelected();
+        }
+
         private void ListContextSelectAllMenu_Click(object sender, EventArgs e)
         {
             SelectAll();
@@ -3512,33 +3617,4 @@ namespace CodeWalker
 
 
 
-    public static class Prompt
-    {
-        public static string ShowDialog(IWin32Window owner, string text, string caption, string defaultvalue = "")
-        {
-            Form prompt = new Form()
-            {
-                Width = 450,
-                Height = 150,
-                FormBorderStyle = FormBorderStyle.FixedDialog,
-                Text = caption,
-                StartPosition = FormStartPosition.CenterParent,
-                MaximizeBox = false,
-                MinimizeBox = false
-            };
-            Label textLabel = new Label() { Left = 30, Top = 20, Width = 370, Height = 20, Text = text, };
-            TextBox textBox = new TextBox() { Left = 30, Top = 40, Width = 370, Text = defaultvalue };
-            Button cancel = new Button() { Text = "Cancel", Left = 230, Width = 80, Top = 70, DialogResult = DialogResult.Cancel };
-            Button confirmation = new Button() { Text = "Ok", Left = 320, Width = 80, Top = 70, DialogResult = DialogResult.OK };
-            cancel.Click += (sender, e) => { prompt.Close(); };
-            confirmation.Click += (sender, e) => { prompt.Close(); };
-            prompt.Controls.Add(textBox);
-            prompt.Controls.Add(confirmation);
-            prompt.Controls.Add(cancel);
-            prompt.Controls.Add(textLabel);
-            prompt.AcceptButton = confirmation;
-
-            return prompt.ShowDialog(owner) == DialogResult.OK ? textBox.Text : "";
-        }
-    }
 }

ExploreForm.resx

@@ -450,24 +450,6 @@
         AQABAQHAAQEBAAEDAcABAQEAAQEBwAEBAQABAwHAAQEBAAEBAQABAQEAAQcBwAEDAQABAQEAAQEBAAEP
         AcABAwEAAQEBAAEBAQABDwHAAQMBAAEBAQABAQEAAQ8BwAEDAQABAQHAAQcBAAEPAcABAwEAAQMB4AEP
         AQABDwHAAQMBAAEHCw==
-</value>
-  </data>
-  <data name="pictureBox1.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
-    <value>
-        iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8
-        YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAACbSURBVDhPlY/BDYAgDEXZxNWcywsLeXECb45A4kX8n1CC
-        UBCaPFNq6H+YVjnnVuAjaxyPFy6dz7F4wj6Ox4qJ97V5v5sA+ykLSZcFUxZlujBskafjGGA/ZFGmY5QW
-        kF+LPF1b0LUo07UFpGlRphOMqwWqhZbeo7LQ0gl+VQbkY9FLby0gyaKV/keywMfOvF+IBlaeYQEHM+Cy
-        MS8nLPK4Pr55DAAAAABJRU5ErkJggg==
-</value>
-  </data>
-  <data name="pictureBox2.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
-    <value>
-        iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8
-        YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAAC1SURBVDhP3ZO9DUIxDISzBw2bkQWYgGnYA+mJhjYzQIWg
-        eKJJG/yBLb34GSGg46Qr4vOd8uMkj1prFg7C5kgta9scIi6FZTje2mp3bovtqSM1NHroVdsTah7X+8vM
-        6Lk5XAkZuxBZFATfbPB1DSlmzmzNN0FDpOlxMgFDdOZ3xIOXgLABGiIN4v2TgJeXaIi06SX+9ow6CyWa
-        QoOvd4MEZPEY5WgaPcNRBhry3WeaQsQPvnNKd1hjo1V/TdpFAAAAAElFTkSuQmCC
 </value>
   </data>
   <metadata name="ListContextMenu.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
@@ -592,6 +574,24 @@
   <metadata name="FolderBrowserDialog.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
     <value>152, 56</value>
   </metadata>
+  <data name="pictureBox1.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+    <value>
+        iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8
+        YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAACbSURBVDhPlY/BDYAgDEXZxNWcywsLeXECb45A4kX8n1CC
+        UBCaPFNq6H+YVjnnVuAjaxyPFy6dz7F4wj6Ox4qJ97V5v5sA+ykLSZcFUxZlujBskafjGGA/ZFGmY5QW
+        kF+LPF1b0LUo07UFpGlRphOMqwWqhZbeo7LQ0gl+VQbkY9FLby0gyaKV/keywMfOvF+IBlaeYQEHM+Cy
+        MS8nLPK4Pr55DAAAAABJRU5ErkJggg==
+</value>
+  </data>
+  <data name="pictureBox2.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+    <value>
+        iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8
+        YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAAC1SURBVDhP3ZO9DUIxDISzBw2bkQWYgGnYA+mJhjYzQIWg
+        eKJJG/yBLb34GSGg46Qr4vOd8uMkj1prFg7C5kgta9scIi6FZTje2mp3bovtqSM1NHroVdsTah7X+8vM
+        6Lk5XAkZuxBZFATfbPB1DSlmzmzNN0FDpOlxMgFDdOZ3xIOXgLABGiIN4v2TgJeXaIi06SX+9ow6CyWa
+        QoOvd4MEZPEY5WgaPcNRBhry3WeaQsQPvnNKd1hjo1V/TdpFAAAAAElFTkSuQmCC
+</value>
+  </data>
   <metadata name="$this.TrayHeight" type="System.Int32, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
     <value>126</value>
   </metadata>

GameFiles/Resources/RpfFile.cs

@@ -1216,7 +1216,7 @@ namespace CodeWalker.GameFiles
             var child = FindChildArchive(f);
             if (child != null)
             {
-                child.StartPos = newstart;
+                child.UpdateStartPos(newstart);
             }
 
         }
@@ -1353,6 +1353,42 @@ namespace CodeWalker.GameFiles
         }
 
 
+        public long GetDefragmentedFileSize()
+        {
+            //this represents the size the file would be when fully defragmented.
+            uint blockcount = GetHeaderBlockCount();
+
+            foreach (var entry in AllEntries)
+            {
+                var fentry = entry as RpfFileEntry;
+                if (fentry != null)
+                {
+                    blockcount += GetBlockCount(fentry.GetFileSize());
+                }
+            }
+
+            return (long)blockcount * 512;
+        }
+
+
+        private void UpdateStartPos(long newpos)
+        {
+            StartPos = newpos;
+
+            if (Children != null)
+            {
+                //make sure children also get their StartPos updated!
+                foreach (var child in Children)
+                {
+                    if (child.ParentFileEntry == null) continue;//shouldn't really happen...
+                    var cpos = StartPos + (long)child.ParentFileEntry.FileOffset * 512;
+                    child.UpdateStartPos(newpos);
+                }
+            }
+        }
+
+
+
 
         public static RpfFile CreateNew(string gtafolder, string relpath, RpfEncryption encryption = RpfEncryption.OPEN)
         {
@@ -1733,6 +1769,92 @@ namespace CodeWalker.GameFiles
         }
 
 
+        public static void Defragment(RpfFile file, Action<string, float> progress)
+        {
+            if (file?.AllEntries == null) return;
+
+            string fpath = file.GetPhysicalFilePath();
+            using (var fstream = File.Open(fpath, FileMode.Open, FileAccess.ReadWrite))
+            {
+                using (var bw = new BinaryWriter(fstream))
+                {
+                    uint destblock = file.GetHeaderBlockCount();
+
+                    const int BUFFER_SIZE = 16384;//what buffer size is best for HDD copy?
+                    var buffer = new byte[BUFFER_SIZE];
+
+                    var allfiles = new List<RpfFileEntry>();
+                    for (int i = 0; i < file.AllEntries.Count; i++)
+                    {
+                        var entry = file.AllEntries[i] as RpfFileEntry;
+                        if (entry != null) allfiles.Add(entry);
+                    }
+                    //make sure we process everything in the current order that they are in the archive
+                    allfiles.Sort((a, b) => { return a.FileOffset.CompareTo(b.FileOffset); });
+
+                    for (int i = 0; i < allfiles.Count; i++)
+                    {
+                        var entry = allfiles[i];
+                        float prog = (float)i / allfiles.Count;
+                        string txt = "Relocating " + entry.Name + "...";
+                        progress(txt, prog);
+
+                        var sourceblock = entry.FileOffset;
+                        var blockcount = GetBlockCount(entry.GetFileSize());
+
+                        if (sourceblock > destblock) //should only be moving things toward the start
+                        {
+                            var source = file.StartPos + (long)sourceblock * 512;
+                            var dest = file.StartPos + (long)destblock * 512;
+                            var remlength = (long)blockcount * 512;
+                            while (remlength > 0)
+                            {
+                                fstream.Position = source;
+                                int n = fstream.Read(buffer, 0, (int)Math.Min(remlength, BUFFER_SIZE));
+                                fstream.Position = dest;
+                                fstream.Write(buffer, 0, n);
+                                source += n;
+                                dest += n;
+                                remlength -= n;
+                            }
+                            entry.FileOffset = destblock;
+
+                            var entryrpf = file.FindChildArchive(entry);
+                            if (entryrpf != null)
+                            {
+                                entryrpf.UpdateStartPos(file.StartPos + (long)entry.FileOffset * 512);
+                            }
+                        }
+                        else if (sourceblock != destblock)
+                        { }//shouldn't get here...
+
+                        destblock += blockcount;
+                    }
+
+                    file.FileSize = (long)destblock * 512;
+
+                    file.WriteHeader(bw);
+
+                    if (file.ParentFileEntry != null)
+                    {
+                        //make sure to also update the parent archive file entry, if there is one
+                        file.ParentFileEntry.FileUncompressedSize = (uint)file.FileSize;
+                        file.ParentFileEntry.FileSize = 0;
+                        if (file.Parent != null)
+                        {
+                            file.Parent.WriteHeader(bw);
+                        }
+                    }
+                    if (file.Parent == null)
+                    {
+                        //this is a root archive, so update the file's length to the new size.
+                        fstream.SetLength(file.FileSize);
+                    }
+                }
+            }
+        }
+
+
 
         private static string GetParentPath(string path)
         {

Utils/Utils.cs

@@ -422,6 +422,48 @@ namespace CodeWalker
 
 
 
+
+
+
+
+
+    public static class Prompt
+    {
+        //handy utility to get a string from the user...
+        public static string ShowDialog(IWin32Window owner, string text, string caption, string defaultvalue = "")
+        {
+            Form prompt = new Form()
+            {
+                Width = 450,
+                Height = 150,
+                FormBorderStyle = FormBorderStyle.FixedDialog,
+                Text = caption,
+                StartPosition = FormStartPosition.CenterParent,
+                MaximizeBox = false,
+                MinimizeBox = false
+            };
+            var textLabel = new Label() { Left = 30, Top = 20, Width = 370, Height = 20, Text = text, };
+            var textBox = new TextBox() { Left = 30, Top = 40, Width = 370, Text = defaultvalue };
+            var cancel = new Button() { Text = "Cancel", Left = 230, Width = 80, Top = 70, DialogResult = DialogResult.Cancel };
+            var confirmation = new Button() { Text = "Ok", Left = 320, Width = 80, Top = 70, DialogResult = DialogResult.OK };
+            cancel.Click += (sender, e) => { prompt.Close(); };
+            confirmation.Click += (sender, e) => { prompt.Close(); };
+            prompt.Controls.Add(textBox);
+            prompt.Controls.Add(confirmation);
+            prompt.Controls.Add(cancel);
+            prompt.Controls.Add(textLabel);
+            prompt.AcceptButton = confirmation;
+
+            return prompt.ShowDialog(owner) == DialogResult.OK ? textBox.Text : "";
+        }
+    }
+
+
+
+
+
+
+
     //unused
     //public class AccurateTimer
     //{