Task/rdmp 365 better load metadata logging

CHANGELOG.md

@@ -4,7 +4,8 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [9.2.1] - Unreleased
+## [9.3.0] - Unreleased
+- Update Load Metadata logging linkage to Catalogue
 - Update Sql Merge component to force destination column types
 
 ## [9.2.0] - 2026-03-10

Rdmp.Core.Tests/CommandExecution/ExecuteCommandCloneLoadMetadataTests.cs

@@ -15,10 +15,7 @@ public void TestCloneLoadMetadata()
         {
             var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd");
             lmd1.Description = "Desc!";
-            var cata = new Catalogue(CatalogueRepository, "myCata")
-            {
-                LoggingDataTask = "B"
-            };
+            var cata = new Catalogue(CatalogueRepository, "myCata");
             cata.SaveToDatabase();
             lmd1.LinkToCatalogue(cata);
             var pt1 = new ProcessTask(CatalogueRepository, lmd1, LoadStage.Mounting)

Rdmp.Core.Tests/CommandExecution/ExecuteCommandConfirmLogsTests.cs

@@ -22,10 +22,7 @@ internal class ExecuteCommandConfirmLogsTests : DatabaseTests
     public void ConfirmLogs_NoEntries_Throws()
     {
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
-        var cata = new Catalogue(CatalogueRepository, "myCata")
-        {
-            LoggingDataTask = "GGG"
-        };
+        var cata = new Catalogue(CatalogueRepository, "myCata");
         cata.SaveToDatabase();
         lmd.LinkToCatalogue(cata);
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
@@ -42,12 +39,9 @@ public void ConfirmLogs_NoEntries_Throws()
     public void ConfirmLogs_HappyEntries_Passes(bool withinTime)
     {
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
-        var cata = new Catalogue(CatalogueRepository, "myCata")
-        {
-            LoggingDataTask = "FFF"
-        };
+        var cata = new Catalogue(CatalogueRepository, "myCata");
         cata.SaveToDatabase();
-        lmd.LinkToCatalogue(cata);
+        lmd.LinkToCatalogue(cata,"FFF");
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
         lm.CreateNewLoggingTaskIfNotExists("FFF");
         var logEntry = lm.CreateDataLoadInfo("FFF", "pack o' cards", "going down gambling", null, true);
@@ -66,12 +60,9 @@ public void ConfirmLogs_HappyEntries_Passes(bool withinTime)
     public void ConfirmLogs_SadEntry_BecauseNeverEnded_Throws()
     {
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
-        var cata = new Catalogue(CatalogueRepository, "myCata")
-        {
-            LoggingDataTask = "FFF"
-        };
+        var cata = new Catalogue(CatalogueRepository, "myCata");
         cata.SaveToDatabase();
-        lmd.LinkToCatalogue(cata);
+        lmd.LinkToCatalogue(cata,"FFF");
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
         lm.CreateNewLoggingTaskIfNotExists("FFF");
 
@@ -88,12 +79,9 @@ public void ConfirmLogs_SadEntry_BecauseNeverEnded_Throws()
     public void ConfirmLogs_SadEntryWithEx_Throws()
     {
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
-        var cata = new Catalogue(CatalogueRepository, "myCata")
-        {
-            LoggingDataTask = "FFF"
-        };
+        var cata = new Catalogue(CatalogueRepository, "myCata");
         cata.SaveToDatabase();
-        lmd.LinkToCatalogue(cata);
+        lmd.LinkToCatalogue(cata,"FFF");
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
         lm.CreateNewLoggingTaskIfNotExists("FFF");
         var logEntry = lm.CreateDataLoadInfo("FFF", "pack o' cards", "going down gambling", null, true);
@@ -110,12 +98,9 @@ public void ConfirmLogs_SadEntryWithEx_Throws()
     public void ConfirmLogs_NotWithinTime_Throws()
     {
         var lmd = new LoadMetadata(CatalogueRepository, "MyLmd");
-        var cata = new Catalogue(CatalogueRepository, "myCata")
-        {
-            LoggingDataTask = "FFF"
-        };
+        var cata = new Catalogue(CatalogueRepository, "myCata");
         cata.SaveToDatabase();
-        lmd.LinkToCatalogue(cata);
+        lmd.LinkToCatalogue(cata,"FFF");
         var lm = new LogManager(lmd.GetDistinctLoggingDatabase());
         lm.CreateNewLoggingTaskIfNotExists("FFF");
         var logEntry = lm.CreateDataLoadInfo("FFF", "pack o' cards", "going down gambling", null, true);
@@ -137,17 +122,11 @@ public void ConfirmLogs_NotWithinTime_Throws()
     public void ConfirmLogs_With2CacheProgress_Throws()
     {
         var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd");
-        var cata = new Catalogue(CatalogueRepository, "myCata")
-        {
-            LoggingDataTask = "B"
-        };
+        var cata = new Catalogue(CatalogueRepository, "myCata");
         cata.SaveToDatabase();
         lmd1.LinkToCatalogue(cata);
         var lmd2 = new LoadMetadata(CatalogueRepository, "MyLmd");
-        var cata2 = new Catalogue(CatalogueRepository, "myCata")
-        {
-            LoggingDataTask = "A"
-        };
+        var cata2 = new Catalogue(CatalogueRepository, "myCata");
         cata2.SaveToDatabase();
         var linkage2 = new LoadMetadataCatalogueLinkage(CatalogueRepository, lmd2, cata2);
         linkage2.SaveToDatabase();

Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateLoadMetadataVersionTests.cs

@@ -15,10 +15,7 @@ public void TestCreateLoadMetadataVersion()
         {
             var lmd1 = new LoadMetadata(CatalogueRepository, "MyLmd");
             lmd1.Description = "Desc!";
-            var cata = new Catalogue(CatalogueRepository, "myCata")
-            {
-                LoggingDataTask = "B"
-            };
+            var cata = new Catalogue(CatalogueRepository, "myCata");
             cata.SaveToDatabase();
             lmd1.LinkToCatalogue(cata);
             var pt1 = new ProcessTask(CatalogueRepository, lmd1, LoadStage.Mounting)

Rdmp.Core.Tests/CommandExecution/ExecuteCommandRestoreLoadMetadataTests.cs

@@ -19,7 +19,6 @@ public void TestRestoreLoadMetadataVersion()
             lmd1.Description = "Desc!";
             var cata = new Catalogue(CatalogueRepository, "myCata")
             {
-                LoggingDataTask = "B"
             };
             cata.SaveToDatabase();
             lmd1.LinkToCatalogue(cata);

Rdmp.Core.Tests/Curation/Integration/LoadProgressUnitTests.cs

@@ -70,12 +70,6 @@ public void LoadProgress_JobFactory_NoDates()
         var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.WorkDirectory),
             "LoadProgress_JobFactory_NoDates", true, (LoadMetadata)lmd);
 
-        foreach (var cata in lmd.GetAllCatalogues())
-        {
-            cata.LoggingDataTask = "ff";
-            cata.SaveToDatabase();
-        }
-
         lmd.SaveToDatabase();
 
         var jobFactory = new SingleScheduledJobFactory(lp, strat, 999, lp.LoadMetadata, null);

Rdmp.Core.Tests/Curation/Integration/MetadataLoggingConfigurationChecksTests.cs

@@ -17,39 +17,6 @@ namespace Rdmp.Core.Tests.Curation.Integration;
 
 public class MetadataLoggingConfigurationChecksTests : UnitTests
 {
-    [Test]
-    public void Test_NoLoggingTask()
-    {
-        var lmd = WhenIHaveA<LoadMetadata>();
-        var cata = WhenIHaveA<Catalogue>();
-        lmd.LinkToCatalogue(cata);
-        Assert.That(lmd.GetAllCatalogues().Count(), Is.EqualTo(2));
-
-        var checks = new MetadataLoggingConfigurationChecks(lmd);
-        var toMem = new ToMemoryCheckNotifier();
-        checks.Check(toMem);
-
-        AssertFailWithFix("Catalogues Mycata,Mycata do not have a logging task specified",
-            "Create a new Logging Task called 'MyLoad'?", toMem);
-    }
-
-    [Test]
-    public void Test_MismatchedLoggingTask()
-    {
-        var lmd = WhenIHaveA<LoadMetadata>();
-        var cata1 = lmd.GetAllCatalogues().Single();
-        var cata2 = WhenIHaveA<Catalogue>();
-        lmd.LinkToCatalogue(cata2);
-        cata1.LoggingDataTask = "OMG YEAGH";
-
-        Assert.That(lmd.GetAllCatalogues().Count(), Is.EqualTo(2));
-
-        var checks = new MetadataLoggingConfigurationChecks(lmd);
-        var toMem = new ToMemoryCheckNotifier();
-        checks.Check(toMem);
-
-        AssertFailWithFix("Some catalogues have NULL LoggingDataTasks", "Set task to OMG YEAGH", toMem);
-    }
 
     [Test]
     public void Test_MissingLoggingServer()
@@ -58,9 +25,7 @@ public void Test_MissingLoggingServer()
         var cata1 = lmd.GetAllCatalogues().Single();
         var cata2 = WhenIHaveA<Catalogue>();
         lmd.LinkToCatalogue(cata2);
-        cata1.LoggingDataTask = "OMG YEAGH";
         cata1.LiveLoggingServer_ID = 2;
-        cata2.LoggingDataTask = "OMG YEAGH";
         cata2.LiveLoggingServer_ID = null;
 
         Assert.That(lmd.GetAllCatalogues().Count(), Is.EqualTo(2));
@@ -84,9 +49,7 @@ public void Test_MissingLoggingServer_UseDefault()
         eds.SaveToDatabase();
 
         lmd.LinkToCatalogue(cata2);
-        cata1.LoggingDataTask = "OMG YEAGH";
         cata1.LiveLoggingServer_ID = null;
-        cata2.LoggingDataTask = "OMG YEAGH";
         cata2.LiveLoggingServer_ID = null;
 
         var defaults = RepositoryLocator.CatalogueRepository;

Rdmp.Core.Tests/Curation/UnitTestsAllObjectsSupported.cs

@@ -35,47 +35,47 @@
 
         var notSupported = new List<Type>();
 
         foreach (var t in types)
         {
             //ignore these types too
-            if (SkipTheseTypes.Contains(t.Name) || t.Name.StartsWith("Spontaneous", StringComparison.Ordinal) ||
+            if (SkipTheseTypes.Contains(t.Name) || t.Name.EndsWith("Proxy") || t.Name.StartsWith("Spontaneous", StringComparison.Ordinal) ||
                 typeof(SpontaneousObject).IsAssignableFrom(t))
                 continue;
 
             DatabaseEntity instance = null;
 
             try
             {
                 //ensure that the method supports the Type
                 var generic = method.MakeGenericMethod(t);
                 instance = (DatabaseEntity)generic.Invoke(this, null);
             }
             catch (TargetInvocationException exception)
             {
                 if (exception.InnerException is TestCaseNotWrittenYetException)
                     notSupported.Add(t);
                 else
                     throw;
             }
 
             //if the instance returned by MakeGenericMethod does not pass checks that's a dealbreaker!
             if (instance != null)
                 try
                 {
                     //and that it returns an instance
                     Assert.That(instance, Is.Not.Null);
                     Assert.Multiple(() =>
                     {
                         Assert.That(instance.Exists());
                         Assert.That(instance.HasLocalChanges().Evaluation, Is.EqualTo(ChangeDescription.NoChanges),
                             "Type was '" + t.Name + "'");
                     });
                 }
                 catch (Exception e)
                 {
                     throw new Exception($"Implementation of WhenIHaveA<{t.Name}> is flawed", e);
                 }
         }
 
         Assert.That(notSupported, Is.Empty,
             $"The following Types were not supported by WhenIHaveA<T>:{Environment.NewLine}{string.Join(Environment.NewLine, notSupported.Select(t => t.Name))}");

Rdmp.Core.Tests/DataLoad/Engine/Integration/CachedFileRetrieverTests.cs

@@ -175,11 +175,15 @@ private ScheduledDataLoadJob CreateTestJob(ILoadDirectory directory)
         var catalogue = Substitute.For<ICatalogue>();
         catalogue.GetTableInfoList(false).Returns(Array.Empty<TableInfo>());
         catalogue.GetLookupTableInfoList().Returns(Array.Empty<TableInfo>());
-        catalogue.LoggingDataTask.Returns("TestLogging");
 
         var logManager = Substitute.For<ILogManager>();
         var loadMetadata = Substitute.For<ILoadMetadata>();
         loadMetadata.GetAllCatalogues().Returns(new[] { catalogue });
+        var link = Substitute.For<ILoadMetadataCatalogueLinkage>();
+        link.CatalogueID.Returns(catalogue.ID);
+        link.LoadMetadataID.Returns(loadMetadata.ID);
+        link.Name.Returns("Test linkage");
+        catalogue.LoggingDataTasks.Returns(new List<ILoadMetadataCatalogueLinkage>() { link });
 
         var j = new ScheduledDataLoadJob(RepositoryLocator, "Test job", logManager, loadMetadata, directory,
             ThrowImmediatelyDataLoadEventListener.Quiet, null)

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataLoadChainerTests.cs

@@ -98,9 +98,9 @@ INSERT [DLCTest] ([AdmissionDate], [DischargeDate], [Condition1], [Condition2],
             var loggingServer = CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
             var logManager = new LogManager(loggingServer);
             logManager.CreateNewLoggingTaskIfNotExists(lmd.Name);
-            cata.LoggingDataTask = lmd.Name;
             cata.SaveToDatabase();
-            lmd.LinkToCatalogue(cata2);
+            lmd.LinkToCatalogue(cata, "MyLoad");
+            lmd.LinkToCatalogue(cata2, "MyLoad");
             var pc = new ProcessTask(CatalogueRepository, lmd, LoadStage.PostLoad);
             pc.ProcessTaskType = ProcessTaskType.DataProvider;
             pc.Path = "Rdmp.Core.DataLoad.Modules.DataProvider.DataLoadChainer";
@@ -126,7 +126,7 @@ INSERT [DLCTest] ([AdmissionDate], [DischargeDate], [Condition1], [Condition2],
             lmd2.LocationOfExecutablesDirectory = Path.GetTempPath();
             lmd2.LocationOfCacheDirectory = Path.GetTempPath();
             lmd2.SaveToDatabase();
-            lmd2.LinkToCatalogue(cata2);
+            lmd2.LinkToCatalogue(cata2, "MyLoad");
             pc = new ProcessTask(CatalogueRepository, lmd2, LoadStage.GetFiles);
             pc.ProcessTaskType = ProcessTaskType.DataProvider;
             pc.Path = "Rdmp.Core.DataLoad.Modules.DataProvider.DoNothingDataProvider";

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataLoadEngineTestsBase.cs

@@ -23,7 +23,7 @@
 /// <summary>
 /// Base class for tests that want to run data loads contains helper methods for setting up a valid DLE load configuration and running it
 /// </summary>
 internal class DataLoadEngineTestsBase : DatabaseTests
 {
     protected static void AssertHasDataLoadRunId(DataRow row)
     {
@@ -80,7 +80,7 @@
 
     protected ITableInfo Import(DiscoveredTable tbl, LoadMetadata lmd, LogManager logManager)
     {
-        logManager.CreateNewLoggingTaskIfNotExists(lmd.Name);
+        logManager.CreateNewLoggingTaskIfNotExists($"Loading {lmd.Name}({lmd.ID})");
 
         //import TableInfos
         var importer = new TableInfoImporter(CatalogueRepository, tbl);
@@ -90,8 +90,6 @@
         var forwardEngineer = new ForwardEngineerCatalogue(ti, cis);
         forwardEngineer.ExecuteForwardEngineering(out var cata, out var cataItems, out var eis);
 
-        //make the catalogue use the load configuration
-        cata.LoggingDataTask = lmd.Name;
         Assert.That(cata.LiveLoggingServer_ID, Is.Not.Null); //catalogue should have one of these because of system defaults
         cata.SaveToDatabase();
         lmd.LinkToCatalogue(cata);

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataLoadProgressUpdateInfoTests.cs

@@ -28,13 +28,17 @@ public class DataLoadProgressUpdateInfoTests : DatabaseTests
     public DataLoadProgressUpdateInfoTests()
     {
         var cata = Substitute.For<ICatalogue>();
-        cata.LoggingDataTask.Returns("NothingTask");
+
         cata.GetTableInfoList(false).Returns(Array.Empty<TableInfo>());
         cata.GetLookupTableInfoList().Returns(Array.Empty<TableInfo>());
 
         var lmd = Substitute.For<ILoadMetadata>();
         lmd.GetAllCatalogues().Returns(new[] { cata });
-
+        var link = Substitute.For<ILoadMetadataCatalogueLinkage>();
+        link.CatalogueID.Returns(cata.ID);
+        link.LoadMetadataID.Returns(lmd.ID);
+        link.Name.Returns("fish");
+        cata.LoggingDataTasks.Returns(new List<ILoadMetadataCatalogueLinkage>() { link });
         _job = new ScheduledDataLoadJob(null, "fish", Substitute.For<ILogManager>(), lmd, null,
             new ThrowImmediatelyDataLoadJob(), null);
     }

Rdmp.Core.Tests/DataLoad/Engine/Integration/HICPipelineTests.cs

@@ -79,10 +79,7 @@ public void Create(CatalogueRepository repository, DiscoveredDatabase database,
             LoadMetadata.LocationOfCacheDirectory = Path.Combine(directory.RootPath.FullName, LoadMetadata.DefaultCachePath);
             LoadMetadata.SaveToDatabase();
 
-            Catalogue = new Catalogue(repository, "HICLoadPipelineTests")
-            {
-                LoggingDataTask = "Test",
-            };
+            Catalogue = new Catalogue(repository, "HICLoadPipelineTests");
             Catalogue.SaveToDatabase();
             LoadMetadata.LinkToCatalogue(Catalogue);
             var catalogueItem = new CatalogueItem(repository, Catalogue, "Test");

Rdmp.Core.Tests/DataLoad/Engine/Integration/PayloadTest.cs

@@ -45,9 +45,8 @@ public void TestPayloadInjection()
 
         MEF.AddTypeToCatalogForTesting(typeof(TestPayloadAttacher));
 
-        b.catalogue.LoggingDataTask = "TestPayloadInjection";
         b.catalogue.SaveToDatabase();
-        lmd.LinkToCatalogue(b.catalogue);
+        lmd.LinkToCatalogue(b.catalogue, "TestPayloadInjection");
         var lm = new LogManager(CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID));
         lm.CreateNewLoggingTaskIfNotExists("TestPayloadInjection");
 

Rdmp.Core.Tests/DataLoad/Modules/Mutilators/RegexRedactionMutilatorTests.cs

@@ -65,6 +65,7 @@ public void RedactionMutilator_BasicTest()
 
         lmd.IgnoreTrigger = true;
         lmd.SaveToDatabase();
+
         var ti = Import(tbl, lmd, logManager);
 
         var projectDirectory = SetupLoadDirectory(lmd);

Rdmp.Core.Tests/Reports/CustomMetadataReportTests.cs

@@ -1387,7 +1387,7 @@ public void TestAllSubs_Catalogue()
 $IsDeprecated
 $IsInternalDataset
 $Last_revision_date
-$LoggingDataTask
+$LoggingDataTasks
 $Name
 $Periodicity
 $PivotCategory_ExtractionInformation_ID

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandAssociateCatalogueWithLoadMetadata.cs

@@ -4,14 +4,15 @@
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
-using System.Linq;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.DataLoad;
 using Rdmp.Core.Icons.IconProvision;
 using Rdmp.Core.Repositories.Construction;
 using Rdmp.Core.ReusableLibraryCode.Icons.IconProvision;
 using SixLabors.ImageSharp;
 using SixLabors.ImageSharp.PixelFormats;
+using System.Linq;
+using System.Threading.Tasks;
 
 namespace Rdmp.Core.CommandExecution.AtomicCommands;
 
@@ -66,36 +67,38 @@ public override void Execute()
             //if there are other catalogues
             if (_otherCatalogues.Any())
             {
-                var tasks = _otherCatalogues.Select(c => c.LoggingDataTask).Distinct().ToArray();
+                var tasks = _otherCatalogues.SelectMany(c => c.LoggingDataTasks).Distinct().ToArray();
                 //if the other catalogues have an agreed logging task
                 if (tasks.Length == 1)
                 {
                     var task = tasks.Single();
 
                     //and that logging task is not blank!, and differs from this Catalogue
-                    if (!string.IsNullOrWhiteSpace(task) && !task.Equals(cata.LoggingDataTask))
+                    if ( !cata.LoggingDataTasks.Select(ldt => ldt.Name).Contains(task.Name))
                     {
                         var liveServers = _otherCatalogues.Where(c => c.LiveLoggingServer_ID != null)
                             .Select(c => c.LiveLoggingServer_ID).Distinct().ToArray();
 
                         //AND if there is agreement on what logging server to use!
                         if (liveServers.Length <= 1)
                             //if there is no current logging task for the Catalogue
-                            if (string.IsNullOrWhiteSpace(cata.LoggingDataTask)
+                            if (!cata.LoggingDataTasks.Any()
                                 //or if the user wants to switch to the new one
                                 || YesNo(
-                                    $"Do you want to set Catalogue '{cata.Name}' to use shared logging task '{task}' instead of its current Logging Task '{cata.LoggingDataTask}' (All Catalogues in a load must share the same task and logging servers)?",
+                                    $"Do you want to set Catalogue '{cata.Name}' to use shared logging task '{task}' instead of its current Logging Task '{cata.LoggingDataTasks.FirstOrDefault()}' (All Catalogues in a load must share the same task and logging servers)?",
                                     "Synchronise Logging Tasks"))
                             {
                                 //switch Catalogue to use that logging task (including servers)
-                                cata.LoggingDataTask = task;
+                                var lmdcl = new LoadMetadataCatalogueLinkage(cata.CatalogueRepository, _loadMetadata, cata,task.Name);
+                                lmdcl.SaveToDatabase();
                                 cata.LiveLoggingServer_ID = liveServers.SingleOrDefault();
                             }
                     }
                 }
                 else if (tasks.Length == 0)
                 {
-                    cata.LoggingDataTask = $"Loading {_loadMetadata.Name}";
+                    var lmdcl = new LoadMetadataCatalogueLinkage(cata.CatalogueRepository, _loadMetadata, cata);
+                    lmdcl.SaveToDatabase();
                 }
             }
             cata.SaveToDatabase();