diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs
index 99117afe35..15225fe821 100644
--- a/osu.Game/Beatmaps/BeatmapManager.cs
+++ b/osu.Game/Beatmaps/BeatmapManager.cs
@@ -19,7 +19,6 @@ using osu.Game.IO;
 using osu.Game.IPC;
 using osu.Game.Rulesets;
 using SQLite.Net;
-using FileInfo = osu.Game.IO.FileInfo;
 
 namespace osu.Game.Beatmaps
 {
@@ -135,7 +134,7 @@ namespace osu.Game.Beatmaps
             if (!beatmaps.Delete(beatmapSet)) return;
 
             if (!beatmapSet.Protected)
-                files.Dereference(beatmapSet.Files);
+                files.Dereference(beatmapSet.Files.Select(f => f.FileInfo));
         }
 
         /// <summary>
@@ -147,7 +146,8 @@ namespace osu.Game.Beatmaps
         {
             if (!beatmaps.Undelete(beatmapSet)) return;
 
-            files.Reference(beatmapSet.Files);
+            if (!beatmapSet.Protected)
+                files.Reference(beatmapSet.Files.Select(f => f.FileInfo));
         }
 
         /// <summary>
@@ -265,12 +265,16 @@ namespace osu.Game.Beatmaps
                 return beatmapSet;
             }
 
-            List<FileInfo> fileInfos = new List<FileInfo>();
+            List<BeatmapSetFileInfo> fileInfos = new List<BeatmapSetFileInfo>();
 
             // import files to manager
             foreach (string file in reader.Filenames)
                 using (Stream s = reader.GetStream(file))
-                    fileInfos.Add(files.Add(s, file));
+                    fileInfos.Add(new BeatmapSetFileInfo
+                    {
+                        Filename = file,
+                        FileInfo = files.Add(s)
+                    });
 
             BeatmapMetadata metadata;
 
@@ -366,7 +370,7 @@ namespace osu.Game.Beatmaps
                 catch { return null; }
             }
 
-            private string getPathForFile(string filename) => BeatmapSetInfo.Files.First(f => f.Filename == filename).StoragePath;
+            private string getPathForFile(string filename) => BeatmapSetInfo.Files.First(f => f.Filename == filename).FileInfo.StoragePath;
 
             protected override Texture GetBackground()
             {
diff --git a/osu.Game/Beatmaps/BeatmapSetFileInfo.cs b/osu.Game/Beatmaps/BeatmapSetFileInfo.cs
index d18b1e833b..a05362b32d 100644
--- a/osu.Game/Beatmaps/BeatmapSetFileInfo.cs
+++ b/osu.Game/Beatmaps/BeatmapSetFileInfo.cs
@@ -2,16 +2,26 @@
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
 using osu.Game.IO;
+using SQLite.Net.Attributes;
 using SQLiteNetExtensions.Attributes;
 
 namespace osu.Game.Beatmaps
 {
     public class BeatmapSetFileInfo
     {
-        [ForeignKey(typeof(BeatmapSetInfo))]
+        [PrimaryKey, AutoIncrement]
+        public int ID { get; set; }
+
+        [ForeignKey(typeof(BeatmapSetInfo)), NotNull]
         public int BeatmapSetInfoID { get; set; }
 
-        [ForeignKey(typeof(FileInfo))]
+        [ForeignKey(typeof(FileInfo)), NotNull]
         public int FileInfoID { get; set; }
+
+        [OneToOne(CascadeOperations = CascadeOperation.CascadeRead)]
+        public FileInfo FileInfo { get; set; }
+
+        [NotNull]
+        public string Filename { get; set; }
     }
 }
\ No newline at end of file
diff --git a/osu.Game/Beatmaps/BeatmapSetInfo.cs b/osu.Game/Beatmaps/BeatmapSetInfo.cs
index a99ba94e9a..f47affcab8 100644
--- a/osu.Game/Beatmaps/BeatmapSetInfo.cs
+++ b/osu.Game/Beatmaps/BeatmapSetInfo.cs
@@ -3,7 +3,6 @@
 
 using System.Collections.Generic;
 using System.Linq;
-using osu.Game.IO;
 using SQLite.Net.Attributes;
 using SQLiteNetExtensions.Attributes;
 
@@ -37,8 +36,8 @@ namespace osu.Game.Beatmaps
 
         public string StoryboardFile => Files.FirstOrDefault(f => f.Filename.EndsWith(".osb"))?.Filename;
 
-        [ManyToMany(typeof(BeatmapSetFileInfo), CascadeOperations = CascadeOperation.CascadeRead)]
-        public List<FileInfo> Files { get; set; }
+        [OneToMany(CascadeOperations = CascadeOperation.All)]
+        public List<BeatmapSetFileInfo> Files { get; set; }
 
         public bool Protected { get; set; }
     }
diff --git a/osu.Game/Database/DatabaseBackedStore.cs b/osu.Game/Database/DatabaseBackedStore.cs
index bb61fc1870..5214372a8b 100644
--- a/osu.Game/Database/DatabaseBackedStore.cs
+++ b/osu.Game/Database/DatabaseBackedStore.cs
@@ -101,7 +101,6 @@ namespace osu.Game.Database
         public void Populate<T>(T item, bool recursive = true)
         {
             checkType(item.GetType());
-
             Connection.GetChildren(item, recursive);
         }
 
diff --git a/osu.Game/IO/FileInfo.cs b/osu.Game/IO/FileInfo.cs
index 6f4c4b26e8..367fd68f7b 100644
--- a/osu.Game/IO/FileInfo.cs
+++ b/osu.Game/IO/FileInfo.cs
@@ -11,8 +11,6 @@ namespace osu.Game.IO
         [PrimaryKey, AutoIncrement]
         public int ID { get; set; }
 
-        public string Filename { get; set; }
-
         [Indexed(Unique = true)]
         public string Hash { get; set; }
 
diff --git a/osu.Game/IO/FileStore.cs b/osu.Game/IO/FileStore.cs
index e8cabafe17..2132037559 100644
--- a/osu.Game/IO/FileStore.cs
+++ b/osu.Game/IO/FileStore.cs
@@ -42,15 +42,11 @@ namespace osu.Game.IO
             deletePending();
         }
 
-        public FileInfo Add(Stream data, string filename = null)
+        public FileInfo Add(Stream data)
         {
             string hash = data.ComputeSHA2Hash();
 
-            var info = new FileInfo
-            {
-                Filename = filename,
-                Hash = hash,
-            };
+            var info = new FileInfo { Hash = hash };
 
             var existing = Connection.Table<FileInfo>().FirstOrDefault(f => f.Hash == info.Hash);
 
@@ -79,20 +75,32 @@ namespace osu.Game.IO
 
         public void Reference(IEnumerable<FileInfo> files)
         {
-            foreach (var f in files)
+            Connection.RunInTransaction(() =>
             {
-                f.ReferenceCount++;
-                Connection.Update(f);
-            }
+                var incrementedFiles = files.GroupBy(f => f.ID).Select(f =>
+                {
+                    var accurateRefCount = Connection.Get<FileInfo>(f.First().ID);
+                    accurateRefCount.ReferenceCount += f.Count();
+                    return accurateRefCount;
+                });
+
+                Connection.UpdateAll(incrementedFiles);
+            });
         }
 
         public void Dereference(IEnumerable<FileInfo> files)
         {
-            foreach (var f in files)
+            Connection.RunInTransaction(() =>
             {
-                f.ReferenceCount--;
-                Connection.Update(f);
-            }
+                var incrementedFiles = files.GroupBy(f => f.ID).Select(f =>
+                {
+                    var accurateRefCount = Connection.Get<FileInfo>(f.First().ID);
+                    accurateRefCount.ReferenceCount -= f.Count();
+                    return accurateRefCount;
+                });
+
+                Connection.UpdateAll(incrementedFiles);
+            });
         }
 
         private void deletePending()