Merge pull request #2673 from peppy/fix-beatmap-import-conflicts

Fix conflicting Hash and Online IDs on beatmap import

osu.Desktop/osu.Desktop.csproj

@@ -20,6 +20,8 @@
     <StartupObject>osu.Desktop.Program</StartupObject>
   </PropertyGroup>
   <ItemGroup Label="Project References">
+    <!-- This can be removed after .NET Core SDK version 2.1.300; see https://docs.microsoft.com/en-us/ef/core/miscellaneous/cli/dotnet -->
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="2.0.3" />
     <ProjectReference Include="..\osu-framework\osu.Framework\osu.Framework.csproj" />
     <ProjectReference Include="..\osu.Game\osu.Game.csproj" />
     <ProjectReference Include="..\osu.Game.Rulesets.Osu\osu.Game.Rulesets.Osu.csproj" />
@@ -36,4 +38,8 @@
   <ItemGroup Label="Resources">
     <EmbeddedResource Include="lazer.ico" />
   </ItemGroup>
+  <ItemGroup>
+    <!-- This can be removed after .NET Core SDK version 2.1.300; see https://docs.microsoft.com/en-us/ef/core/miscellaneous/cli/dotnet -->
+    <DotNetCliToolReference Include="Microsoft.EntityFrameworkCore.Tools.DotNet" Version="2.0.0" />
+  </ItemGroup>
 </Project>

osu.Game/Beatmaps/BeatmapManager.cs

@@ -81,7 +81,7 @@ namespace osu.Game.Beatmaps
 
         protected override void Populate(BeatmapSetInfo model, ArchiveReader archive)
         {
-            model.Beatmaps = createBeatmapDifficulties(archive);
+            model.Beatmaps = createBeatmapDifficulties(model, archive);
 
             // remove metadata from difficulties where it matches the set
             foreach (BeatmapInfo b in model.Beatmaps)
@@ -107,6 +107,7 @@ namespace osu.Game.Beatmaps
                 {
                     Delete(existingOnlineId);
                     beatmaps.PurgeDeletable(s => s.ID == existingOnlineId.ID);
+                    Logger.Log($"Found existing beatmap set with same OnlineBeatmapSetID ({model.OnlineBeatmapSetID}). It has been purged.", LoggingTarget.Database);
                 }
             }
 
@@ -303,7 +304,7 @@ namespace osu.Game.Beatmaps
         {
             // let's make sure there are actually .osu files to import.
             string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu"));
-            if (string.IsNullOrEmpty(mapName)) throw new InvalidOperationException("No beatmap files found in the map folder.");
+            if (string.IsNullOrEmpty(mapName)) throw new InvalidOperationException("No beatmap files found in this beatmap archive.");
 
             BeatmapMetadata metadata;
             using (var stream = new StreamReader(reader.GetStream(mapName)))
@@ -321,7 +322,7 @@ namespace osu.Game.Beatmaps
         /// <summary>
         /// Create all required <see cref="BeatmapInfo"/>s for the provided archive.
         /// </summary>
-        private List<BeatmapInfo> createBeatmapDifficulties(ArchiveReader reader)
+        private List<BeatmapInfo> createBeatmapDifficulties(BeatmapSetInfo model, ArchiveReader reader)
         {
             var beatmapInfos = new List<BeatmapInfo>();
 
@@ -341,6 +342,14 @@ namespace osu.Game.Beatmaps
                     beatmap.BeatmapInfo.Hash = ms.ComputeSHA2Hash();
                     beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash();
 
+                    // ensure we have the same online set ID as the set itself.
+                    beatmap.BeatmapInfo.OnlineBeatmapSetID = model.OnlineBeatmapSetID;
+                    beatmap.BeatmapInfo.Metadata.OnlineBeatmapSetID = model.OnlineBeatmapSetID;
+
+                    // check that no existing beatmap exists that is imported with the same online beatmap ID. if so, give it precedence.
+                    if (beatmap.BeatmapInfo.OnlineBeatmapID.HasValue && QueryBeatmap(b => b.OnlineBeatmapID.Value == beatmap.BeatmapInfo.OnlineBeatmapID.Value) != null)
+                        beatmap.BeatmapInfo.OnlineBeatmapID = null;
+
                     RulesetInfo ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID);
 
                     beatmap.BeatmapInfo.Ruleset = ruleset;

osu.Game/Database/ArchiveModelManager.cs

@@ -190,7 +190,11 @@ namespace osu.Game.Database
 
                         var existing = CheckForExisting(item);
 
-                        if (existing != null) return existing;
+                        if (existing != null)
+                        {
+                            Logger.Log($"Found existing {typeof(TModel)} for {archive.Name} (ID {existing.ID}). Skipping import.", LoggingTarget.Database);
+                            return existing;
+                        }
 
                         item.Files = createFileInfos(archive, Files);
 
@@ -205,6 +209,8 @@ namespace osu.Game.Database
                         throw;
                     }
                 }
+
+                Logger.Log($"Import of {archive.Name} successfully completed!", LoggingTarget.Database);
             }
             catch (Exception e)
             {

osu.Game/Database/OsuDbContext.cs

@@ -82,8 +82,8 @@ namespace osu.Game.Database
             base.OnModelCreating(modelBuilder);
 
             modelBuilder.Entity<BeatmapInfo>().HasIndex(b => b.OnlineBeatmapID).IsUnique();
-            modelBuilder.Entity<BeatmapInfo>().HasIndex(b => b.MD5Hash).IsUnique();
-            modelBuilder.Entity<BeatmapInfo>().HasIndex(b => b.Hash).IsUnique();
+            modelBuilder.Entity<BeatmapInfo>().HasIndex(b => b.MD5Hash);
+            modelBuilder.Entity<BeatmapInfo>().HasIndex(b => b.Hash);
 
             modelBuilder.Entity<BeatmapSetInfo>().HasIndex(b => b.OnlineBeatmapSetID).IsUnique();
             modelBuilder.Entity<BeatmapSetInfo>().HasIndex(b => b.DeletePending);

osu.Game/Migrations/20180529055154_RemoveUniqueHashConstraints.Designer.cs

@@ -0,0 +1,377 @@
+// <auto-generated />
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage;
+using osu.Game.Database;
+using System;
+
+namespace osu.Game.Migrations
+{
+    [DbContext(typeof(OsuDbContext))]
+    [Migration("20180529055154_RemoveUniqueHashConstraints")]
+    partial class RemoveUniqueHashConstraints
+    {
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "2.0.3-rtm-10026");
+
+            modelBuilder.Entity("osu.Game.Beatmaps.BeatmapDifficulty", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<float>("ApproachRate");
+
+                    b.Property<float>("CircleSize");
+
+                    b.Property<float>("DrainRate");
+
+                    b.Property<float>("OverallDifficulty");
+
+                    b.Property<double>("SliderMultiplier");
+
+                    b.Property<double>("SliderTickRate");
+
+                    b.HasKey("ID");
+
+                    b.ToTable("BeatmapDifficulty");
+                });
+
+            modelBuilder.Entity("osu.Game.Beatmaps.BeatmapInfo", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<int>("AudioLeadIn");
+
+                    b.Property<int>("BaseDifficultyID");
+
+                    b.Property<int>("BeatDivisor");
+
+                    b.Property<int>("BeatmapSetInfoID");
+
+                    b.Property<bool>("Countdown");
+
+                    b.Property<double>("DistanceSpacing");
+
+                    b.Property<int>("GridSize");
+
+                    b.Property<string>("Hash");
+
+                    b.Property<bool>("Hidden");
+
+                    b.Property<bool>("LetterboxInBreaks");
+
+                    b.Property<string>("MD5Hash");
+
+                    b.Property<int?>("MetadataID");
+
+                    b.Property<int?>("OnlineBeatmapID");
+
+                    b.Property<string>("Path");
+
+                    b.Property<int>("RulesetID");
+
+                    b.Property<bool>("SpecialStyle");
+
+                    b.Property<float>("StackLeniency");
+
+                    b.Property<double>("StarDifficulty");
+
+                    b.Property<string>("StoredBookmarks");
+
+                    b.Property<double>("TimelineZoom");
+
+                    b.Property<string>("Version");
+
+                    b.Property<bool>("WidescreenStoryboard");
+
+                    b.HasKey("ID");
+
+                    b.HasIndex("BaseDifficultyID");
+
+                    b.HasIndex("BeatmapSetInfoID");
+
+                    b.HasIndex("Hash");
+
+                    b.HasIndex("MD5Hash");
+
+                    b.HasIndex("MetadataID");
+
+                    b.HasIndex("OnlineBeatmapID")
+                        .IsUnique();
+
+                    b.HasIndex("RulesetID");
+
+                    b.ToTable("BeatmapInfo");
+                });
+
+            modelBuilder.Entity("osu.Game.Beatmaps.BeatmapMetadata", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<string>("Artist");
+
+                    b.Property<string>("ArtistUnicode");
+
+                    b.Property<string>("AudioFile");
+
+                    b.Property<string>("AuthorString")
+                        .HasColumnName("Author");
+
+                    b.Property<string>("BackgroundFile");
+
+                    b.Property<int>("PreviewTime");
+
+                    b.Property<string>("Source");
+
+                    b.Property<string>("Tags");
+
+                    b.Property<string>("Title");
+
+                    b.Property<string>("TitleUnicode");
+
+                    b.HasKey("ID");
+
+                    b.ToTable("BeatmapMetadata");
+                });
+
+            modelBuilder.Entity("osu.Game.Beatmaps.BeatmapSetFileInfo", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<int>("BeatmapSetInfoID");
+
+                    b.Property<int>("FileInfoID");
+
+                    b.Property<string>("Filename")
+                        .IsRequired();
+
+                    b.HasKey("ID");
+
+                    b.HasIndex("BeatmapSetInfoID");
+
+                    b.HasIndex("FileInfoID");
+
+                    b.ToTable("BeatmapSetFileInfo");
+                });
+
+            modelBuilder.Entity("osu.Game.Beatmaps.BeatmapSetInfo", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<bool>("DeletePending");
+
+                    b.Property<string>("Hash");
+
+                    b.Property<int?>("MetadataID");
+
+                    b.Property<int?>("OnlineBeatmapSetID");
+
+                    b.Property<bool>("Protected");
+
+                    b.HasKey("ID");
+
+                    b.HasIndex("DeletePending");
+
+                    b.HasIndex("Hash")
+                        .IsUnique();
+
+                    b.HasIndex("MetadataID");
+
+                    b.HasIndex("OnlineBeatmapSetID")
+                        .IsUnique();
+
+                    b.ToTable("BeatmapSetInfo");
+                });
+
+            modelBuilder.Entity("osu.Game.Configuration.DatabasedSetting", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<int>("IntKey")
+                        .HasColumnName("Key");
+
+                    b.Property<int?>("RulesetID");
+
+                    b.Property<string>("StringValue")
+                        .HasColumnName("Value");
+
+                    b.Property<int?>("Variant");
+
+                    b.HasKey("ID");
+
+                    b.HasIndex("RulesetID", "Variant");
+
+                    b.ToTable("Settings");
+                });
+
+            modelBuilder.Entity("osu.Game.Input.Bindings.DatabasedKeyBinding", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<int>("IntAction")
+                        .HasColumnName("Action");
+
+                    b.Property<string>("KeysString")
+                        .HasColumnName("Keys");
+
+                    b.Property<int?>("RulesetID");
+
+                    b.Property<int?>("Variant");
+
+                    b.HasKey("ID");
+
+                    b.HasIndex("IntAction");
+
+                    b.HasIndex("RulesetID", "Variant");
+
+                    b.ToTable("KeyBinding");
+                });
+
+            modelBuilder.Entity("osu.Game.IO.FileInfo", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<string>("Hash");
+
+                    b.Property<int>("ReferenceCount");
+
+                    b.HasKey("ID");
+
+                    b.HasIndex("Hash")
+                        .IsUnique();
+
+                    b.HasIndex("ReferenceCount");
+
+                    b.ToTable("FileInfo");
+                });
+
+            modelBuilder.Entity("osu.Game.Rulesets.RulesetInfo", b =>
+                {
+                    b.Property<int?>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<bool>("Available");
+
+                    b.Property<string>("InstantiationInfo");
+
+                    b.Property<string>("Name");
+
+                    b.Property<string>("ShortName");
+
+                    b.HasKey("ID");
+
+                    b.HasIndex("Available");
+
+                    b.HasIndex("ShortName")
+                        .IsUnique();
+
+                    b.ToTable("RulesetInfo");
+                });
+
+            modelBuilder.Entity("osu.Game.Skinning.SkinFileInfo", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<int>("FileInfoID");
+
+                    b.Property<string>("Filename")
+                        .IsRequired();
+
+                    b.Property<int>("SkinInfoID");
+
+                    b.HasKey("ID");
+
+                    b.HasIndex("FileInfoID");
+
+                    b.HasIndex("SkinInfoID");
+
+                    b.ToTable("SkinFileInfo");
+                });
+
+            modelBuilder.Entity("osu.Game.Skinning.SkinInfo", b =>
+                {
+                    b.Property<int>("ID")
+                        .ValueGeneratedOnAdd();
+
+                    b.Property<string>("Creator");
+
+                    b.Property<bool>("DeletePending");
+
+                    b.Property<string>("Name");
+
+                    b.HasKey("ID");
+
+                    b.ToTable("SkinInfo");
+                });
+
+            modelBuilder.Entity("osu.Game.Beatmaps.BeatmapInfo", b =>
+                {
+                    b.HasOne("osu.Game.Beatmaps.BeatmapDifficulty", "BaseDifficulty")
+                        .WithMany()
+                        .HasForeignKey("BaseDifficultyID")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("osu.Game.Beatmaps.BeatmapSetInfo", "BeatmapSet")
+                        .WithMany("Beatmaps")
+                        .HasForeignKey("BeatmapSetInfoID")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("osu.Game.Beatmaps.BeatmapMetadata", "Metadata")
+                        .WithMany("Beatmaps")
+                        .HasForeignKey("MetadataID");
+
+                    b.HasOne("osu.Game.Rulesets.RulesetInfo", "Ruleset")
+                        .WithMany()
+                        .HasForeignKey("RulesetID")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("osu.Game.Beatmaps.BeatmapSetFileInfo", b =>
+                {
+                    b.HasOne("osu.Game.Beatmaps.BeatmapSetInfo")
+                        .WithMany("Files")
+                        .HasForeignKey("BeatmapSetInfoID")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("osu.Game.IO.FileInfo", "FileInfo")
+                        .WithMany()
+                        .HasForeignKey("FileInfoID")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+
+            modelBuilder.Entity("osu.Game.Beatmaps.BeatmapSetInfo", b =>
+                {
+                    b.HasOne("osu.Game.Beatmaps.BeatmapMetadata", "Metadata")
+                        .WithMany("BeatmapSets")
+                        .HasForeignKey("MetadataID");
+                });
+
+            modelBuilder.Entity("osu.Game.Skinning.SkinFileInfo", b =>
+                {
+                    b.HasOne("osu.Game.IO.FileInfo", "FileInfo")
+                        .WithMany()
+                        .HasForeignKey("FileInfoID")
+                        .OnDelete(DeleteBehavior.Cascade);
+
+                    b.HasOne("osu.Game.Skinning.SkinInfo")
+                        .WithMany("Files")
+                        .HasForeignKey("SkinInfoID")
+                        .OnDelete(DeleteBehavior.Cascade);
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

osu.Game/Migrations/20180529055154_RemoveUniqueHashConstraints.cs

@@ -0,0 +1,53 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+using System;
+using System.Collections.Generic;
+
+namespace osu.Game.Migrations
+{
+    public partial class RemoveUniqueHashConstraints : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropIndex(
+                name: "IX_BeatmapInfo_Hash",
+                table: "BeatmapInfo");
+
+            migrationBuilder.DropIndex(
+                name: "IX_BeatmapInfo_MD5Hash",
+                table: "BeatmapInfo");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_BeatmapInfo_Hash",
+                table: "BeatmapInfo",
+                column: "Hash");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_BeatmapInfo_MD5Hash",
+                table: "BeatmapInfo",
+                column: "MD5Hash");
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropIndex(
+                name: "IX_BeatmapInfo_Hash",
+                table: "BeatmapInfo");
+
+            migrationBuilder.DropIndex(
+                name: "IX_BeatmapInfo_MD5Hash",
+                table: "BeatmapInfo");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_BeatmapInfo_Hash",
+                table: "BeatmapInfo",
+                column: "Hash",
+                unique: true);
+
+            migrationBuilder.CreateIndex(
+                name: "IX_BeatmapInfo_MD5Hash",
+                table: "BeatmapInfo",
+                column: "MD5Hash",
+                unique: true);
+        }
+    }
+}

osu.Game/Migrations/OsuDbContextModelSnapshot.cs

@@ -1,7 +1,11 @@
 // <auto-generated />
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage;
 using osu.Game.Database;
+using System;
 
 namespace osu.Game.Migrations
 {
@@ -12,7 +16,7 @@ namespace osu.Game.Migrations
         {
 #pragma warning disable 612, 618
             modelBuilder
-                .HasAnnotation("ProductVersion", "2.0.0-rtm-26452");
+                .HasAnnotation("ProductVersion", "2.0.3-rtm-10026");
 
             modelBuilder.Entity("osu.Game.Beatmaps.BeatmapDifficulty", b =>
                 {
@@ -27,9 +31,9 @@ namespace osu.Game.Migrations
 
                     b.Property<float>("OverallDifficulty");
 
-                    b.Property<float>("SliderMultiplier");
+                    b.Property<double>("SliderMultiplier");
 
-                    b.Property<float>("SliderTickRate");
+                    b.Property<double>("SliderTickRate");
 
                     b.HasKey("ID");
 
@@ -91,11 +95,9 @@ namespace osu.Game.Migrations
 
                     b.HasIndex("BeatmapSetInfoID");
 
-                    b.HasIndex("Hash")
-                        .IsUnique();
+                    b.HasIndex("Hash");
 
-                    b.HasIndex("MD5Hash")
-                        .IsUnique();
+                    b.HasIndex("MD5Hash");
 
                     b.HasIndex("MetadataID");