From acfae60734af49a06f916c96d86d0c5e63d66e29 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 3 Jun 2026 14:19:30 +0100 Subject: [PATCH 1/8] working add --- .../ExecuteFullExtractionToDatabaseMSSql.cs | 85 +++++++++++++++---- .../MicrosoftSQLTriggerImplementer.cs | 18 +++- 2 files changed, 84 insertions(+), 19 deletions(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index 6d8e95745e..4b3a0ed837 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -6,6 +6,7 @@ using Amazon.Auth.AccessControlPolicy; using FAnsi.Discovery; +using NPOI.SS.Formula.Functions; using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; @@ -16,6 +17,7 @@ using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataLoad.Engine.Job.Scheduling; using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; +using Rdmp.Core.DataLoad.Triggers; using Rdmp.Core.DataLoad.Triggers.Exceptions; using Rdmp.Core.DataLoad.Triggers.Implementations; using Rdmp.Core.MapsDirectlyToDatabaseTable; @@ -169,6 +171,14 @@ protected override void WriteRows(DataTable toProcess, IDataLoadEventListener jo LinesWritten += toProcess.Rows.Count; } + + private bool hasStructuralChanges(DataTable source, DiscoveredTable destination) + { + var sourceColumns = source.Columns.Cast().Select(c => c.ColumnName).ToList(); + var destinationColumns = destination.DiscoverColumns().Select(c => c.GetRuntimeName()).ToList(); + return !sourceColumns.All(destinationColumns.Contains) || !destinationColumns.All(sourceColumns.Contains); + } + private DataTableUploadDestination PrepareDestination(IDataLoadEventListener listener, DataTable toProcess) { //see if the user has entered an extraction server/database @@ -191,7 +201,17 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis if (existing.Exists()) { var hasPKs = existing.DiscoverColumns().Any(col => col.IsPrimaryKey); - + TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer); + var implementor = triggerFactory.Create(existing); + bool present; + try + { + present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + } + catch (TriggerMissingException) + { + present = false; + } if (!AlwaysDropExtractionTables) { //check the PKs are the same @@ -205,7 +225,51 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis Source PKs: {string.Join(", ", rdmpPKs)} Destination PKs: {string.Join(", ", remotePKs)} """)); - return null; + return null;//todo this error could be better + } + if (hasStructuralChanges(toProcess, existing)) + { + var sourceColumns = toProcess.Columns.Cast().Select(c => c.ColumnName).ToList(); + var destinationColumns = existing.DiscoverColumns().Select(c => c.GetRuntimeName()).ToList(); + + //no way to create new archive trigger based on new columns if the archive has columns that aren't in the + + if (present && destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)).Any())//only mess about with column removal if there is an archive trigger + { + + //move everything into the archive - do this by updating the HIC_validfrom + var sql = $"UPDATE {existing.GetFullyQualifiedName()} set {SpecialFieldNames.ValidFrom} = GETDATE()"; + using var con = _destinationDatabase.Server.GetConnection(); + con.Open(); + using var cmd = _destinationDatabase.Server.GetCommand(sql, con); + cmd.CommandTimeout = 30000; + cmd.ExecuteNonQuery(); + + var removedColumns = destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)); + foreach (var column in removedColumns) + { + var discoveredColumn = existing.DiscoverColumn(column); + existing.DropColumn(discoveredColumn); + } + string triggerProblems = ""; + string triggerOK = ""; + implementor.DropTrigger(out triggerProblems, out triggerOK); + if (triggerProblems != "") + { + listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, triggerProblems)); + } + + existing = _destinationDatabase.ExpectTable(tblName); + implementor = triggerFactory.Create(existing); + try + { + present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + } + catch (TriggerMissingException) + { + present = false; + } + } } } @@ -229,18 +293,6 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis } else if (UseArchiveTrigger && hasPKs) { - - TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer); - var implementor = triggerFactory.Create(existing); - bool present; - try - { - present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; - } - catch (TriggerMissingException) - { - present = false; - } //check the columns are correct, we might have added some var existingColumns = existing.DiscoverColumns(); var existingColumnNames = existingColumns.Select(ec => ec.GetRuntimeName()); @@ -254,7 +306,10 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis foreach (var column in newColumns) { existing.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000); - archiveTable.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000); + if (archiveTable.DiscoverColumns().All(col => col.GetRuntimeName() != column)) + { + archiveTable.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000); + } } if (present) { diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs index a4eb823784..fbc5e407c5 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs @@ -270,10 +270,20 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun += $"BEGIN{Environment.NewLine}"; sqlToRun += Environment.NewLine; - var liveCols = _columns.Select(c => $"[{c.GetRuntimeName()}]").Union(new string[] - { - $"[{SpecialFieldNames.DataLoadRunID}]", $"[{SpecialFieldNames.ValidFrom}]" - }).Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true).ToArray(); + + //var liveCols = _columns.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]").Union(new string[] + var liveCols = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]") + .Where(c => c != "[hic_validTo]" && c != "[hic_userID]" && c != "[hic_status]") + .ToList(); + //.Union(new string[] + //{ + //$"[{SpecialFieldNames.DataLoadRunID}]", $"[{SpecialFieldNames.ValidFrom}]" + //}).Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true) + //.Where(col => col != "[hic_validTo]" && col != "hic_userID" && col != "hic_status") + //.ToArray(); + if (!liveCols.Contains($"[{SpecialFieldNames.DataLoadRunID}]")) liveCols.Add($"[{SpecialFieldNames.DataLoadRunID}]"); + if (!liveCols.Contains($"[{SpecialFieldNames.ValidFrom}]")) liveCols.Add($"[{SpecialFieldNames.ValidFrom}]"); + liveCols = liveCols.Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true).ToList(); var archiveCols = $"{string.Join(",", liveCols)},hic_validTo,hic_userID,hic_status"; var cDotArchiveCols = string.Join(",", liveCols.Select(s => $"c.{s}")); From ba4865d7876633314aff0e01b9f3668007023cc5 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 4 Jun 2026 08:58:59 +0100 Subject: [PATCH 2/8] col working --- .../MicrosoftSQLTriggerImplementer.cs | 26 +++++++++++++++---- .../Implementations/TriggerImplementer.cs | 5 ++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs index fbc5e407c5..d81964e280 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs @@ -259,10 +259,19 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch //these were added during transaction so we have to specify them again here because transaction will not have been committed yet sqlToRun = sqlToRun.Trim(); - sqlToRun += $",{Environment.NewLine}"; - sqlToRun += $"\thic_validTo datetime,{Environment.NewLine}"; - sqlToRun += "\thic_userID varchar(128),"; - sqlToRun += "\thic_status char(1)"; + if (!sqlToRun.Contains("hic_validTo")) + { + sqlToRun += $"{Environment.NewLine}"; + sqlToRun += $",\thic_validTo datetime"; + } + if (!sqlToRun.Contains("hic_userID")) + { + sqlToRun += ",\thic_userID varchar(128)"; + } + if (!sqlToRun.Contains("hic_status")) + { + sqlToRun += ",\thic_status char(1)"; + } sqlToRun += $"){Environment.NewLine}"; @@ -285,7 +294,12 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch if (!liveCols.Contains($"[{SpecialFieldNames.ValidFrom}]")) liveCols.Add($"[{SpecialFieldNames.ValidFrom}]"); liveCols = liveCols.Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true).ToList(); - var archiveCols = $"{string.Join(",", liveCols)},hic_validTo,hic_userID,hic_status"; + var archiveCols = $"{string.Join(",", liveCols)}"; + //$",hic_validTo,hic_userID,hic_status"; + if (!archiveCols.Contains("hic_validTo")) archiveCols += ",hic_validTo"; + if (!archiveCols.Contains("hic_userID")) archiveCols += ",hic_userID "; + if (!archiveCols.Contains("hic_status")) archiveCols += ",hic_status "; + var cDotArchiveCols = string.Join(",", liveCols.Select(s => $"c.{s}")); @@ -310,6 +324,8 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun += $"\tAND{Environment.NewLine}"; //add an AND because there are more coming } + //TODO: Current issue is that columsn don;t exist in the destination table, but inthe archive, need to set something like NULL as Interpretation + sqlToRun += string.Format("\tWHERE a.[{0}] IS NULL -- where archive record doesn't exist" + Environment.NewLine, _primaryKeys.First().GetRuntimeName()); sqlToRun += $"\tAND @index > ISNULL(c.{SpecialFieldNames.ValidFrom}, '1899/01/01'){Environment.NewLine}"; diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/TriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/TriggerImplementer.cs index 95124fc55d..5b7313a08a 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/TriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/TriggerImplementer.cs @@ -136,9 +136,10 @@ protected virtual void AddValidFrom(DiscoveredTable table, IQuerySyntaxHelper sy private string WorkOutArchiveTableCreationSQL() { //script original table - var createTableSQL = _table.ScriptTableCreation(true, true, true); + var tbl = _archiveTable.Exists() ? _archiveTable : _table; + var createTableSQL = tbl.ScriptTableCreation(true, true, true); - var toReplaceTableName = $"CREATE TABLE {_table.GetFullyQualifiedName()}"; + var toReplaceTableName = $"CREATE TABLE {tbl.GetFullyQualifiedName()}"; if (!createTableSQL.Contains(toReplaceTableName)) throw new Exception($"Expected to find occurrence of {toReplaceTableName} in the SQL {createTableSQL}"); From 0a6e3f39f32b93418787eea3143d1d734d375bd0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 4 Jun 2026 09:32:50 +0100 Subject: [PATCH 3/8] null columns --- .../MicrosoftSQLTriggerImplementer.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs index d81964e280..e6fb9520cd 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs @@ -280,16 +280,12 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun += Environment.NewLine; + var x = _archiveTable.DiscoverColumns(); //var liveCols = _columns.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]").Union(new string[] var liveCols = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]") .Where(c => c != "[hic_validTo]" && c != "[hic_userID]" && c != "[hic_status]") .ToList(); - //.Union(new string[] - //{ - //$"[{SpecialFieldNames.DataLoadRunID}]", $"[{SpecialFieldNames.ValidFrom}]" - //}).Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true) - //.Where(col => col != "[hic_validTo]" && col != "hic_userID" && col != "hic_status") - //.ToArray(); + if (!liveCols.Contains($"[{SpecialFieldNames.DataLoadRunID}]")) liveCols.Add($"[{SpecialFieldNames.DataLoadRunID}]"); if (!liveCols.Contains($"[{SpecialFieldNames.ValidFrom}]")) liveCols.Add($"[{SpecialFieldNames.ValidFrom}]"); liveCols = liveCols.Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true).ToList(); @@ -309,6 +305,9 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch ", '1899/01/01') AND hic_validTo" + Environment.NewLine, _archiveTable); sqlToRun += Environment.NewLine; + var nullCoulmns = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]").Except(_table.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]")); + cDotArchiveCols = string.Join(",", liveCols.Select(s => nullCoulmns.Contains(s) ? $"NULL AS {s}" : $"c.{s}")); + sqlToRun += $"\tINSERT @returntable{Environment.NewLine}"; sqlToRun += $"\tSELECT {cDotArchiveCols},NULL AS hic_validTo, NULL AS hic_userID, 'C' AS hic_status{Environment.NewLine}"; //c is for current @@ -324,7 +323,7 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun += $"\tAND{Environment.NewLine}"; //add an AND because there are more coming } - //TODO: Current issue is that columsn don;t exist in the destination table, but inthe archive, need to set something like NULL as Interpretation + //TODO: ordering issue when adding a new column and removing an existing one sqlToRun += string.Format("\tWHERE a.[{0}] IS NULL -- where archive record doesn't exist" + Environment.NewLine, _primaryKeys.First().GetRuntimeName()); From c24f37dad0ba9250205783b2962616f8df8ff1f7 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 4 Jun 2026 13:57:34 +0100 Subject: [PATCH 4/8] working triggr --- .../MicrosoftSQLTriggerImplementer.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs index e6fb9520cd..3fb6c20830 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs @@ -280,10 +280,10 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun += Environment.NewLine; - var x = _archiveTable.DiscoverColumns(); + //var x = _archiveTable.DiscoverColumns(); //var liveCols = _columns.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]").Union(new string[] var liveCols = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]") - .Where(c => c != "[hic_validTo]" && c != "[hic_userID]" && c != "[hic_status]") + //.Where(c => c != "[hic_validTo]" && c != "[hic_userID]" && c != "[hic_status]") .ToList(); if (!liveCols.Contains($"[{SpecialFieldNames.DataLoadRunID}]")) liveCols.Add($"[{SpecialFieldNames.DataLoadRunID}]"); @@ -292,9 +292,9 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch var archiveCols = $"{string.Join(",", liveCols)}"; //$",hic_validTo,hic_userID,hic_status"; - if (!archiveCols.Contains("hic_validTo")) archiveCols += ",hic_validTo"; - if (!archiveCols.Contains("hic_userID")) archiveCols += ",hic_userID "; - if (!archiveCols.Contains("hic_status")) archiveCols += ",hic_status "; + //if (!archiveCols.Contains("hic_validTo")) archiveCols += ",hic_validTo"; + //if (!archiveCols.Contains("hic_userID")) archiveCols += ",hic_userID "; + //if (!archiveCols.Contains("hic_status")) archiveCols += ",hic_status "; var cDotArchiveCols = string.Join(",", liveCols.Select(s => $"c.{s}")); @@ -310,7 +310,7 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun += $"\tINSERT @returntable{Environment.NewLine}"; sqlToRun += - $"\tSELECT {cDotArchiveCols},NULL AS hic_validTo, NULL AS hic_userID, 'C' AS hic_status{Environment.NewLine}"; //c is for current + $"\tSELECT {cDotArchiveCols}";//,NULL AS hic_validTo, NULL AS hic_userID, 'C' AS hic_status{Environment.NewLine}"; //c is for current sqlToRun += string.Format("\tFROM [{0}] c" + Environment.NewLine, _table.GetRuntimeName()); sqlToRun += $"\tLEFT OUTER JOIN @returntable a ON {Environment.NewLine}"; From 2cfbe0759f2129b13c0647844798f57df8a71250 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 4 Jun 2026 15:46:07 +0100 Subject: [PATCH 5/8] tidy up --- .../ExecuteFullExtractionToDatabaseMSSql.cs | 21 +++++++++---------- .../MicrosoftSQLTriggerImplementer.cs | 9 -------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index 4b3a0ed837..aa7f94205b 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -179,6 +179,7 @@ private bool hasStructuralChanges(DataTable source, DiscoveredTable destination) return !sourceColumns.All(destinationColumns.Contains) || !destinationColumns.All(sourceColumns.Contains); } + private DataTableUploadDestination PrepareDestination(IDataLoadEventListener listener, DataTable toProcess) { //see if the user has entered an extraction server/database @@ -203,14 +204,14 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis var hasPKs = existing.DiscoverColumns().Any(col => col.IsPrimaryKey); TriggerImplementerFactory triggerFactory = new TriggerImplementerFactory(FAnsi.DatabaseType.MicrosoftSQLServer); var implementor = triggerFactory.Create(existing); - bool present; + bool triggerPresent; try { - present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + triggerPresent = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; } catch (TriggerMissingException) { - present = false; + triggerPresent = false; } if (!AlwaysDropExtractionTables) { @@ -232,9 +233,7 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis var sourceColumns = toProcess.Columns.Cast().Select(c => c.ColumnName).ToList(); var destinationColumns = existing.DiscoverColumns().Select(c => c.GetRuntimeName()).ToList(); - //no way to create new archive trigger based on new columns if the archive has columns that aren't in the - - if (present && destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)).Any())//only mess about with column removal if there is an archive trigger + if (triggerPresent && destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)).Any())//only mess about with column removal if there is an archive trigger { //move everything into the archive - do this by updating the HIC_validfrom @@ -263,11 +262,11 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis implementor = triggerFactory.Create(existing); try { - present = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; + triggerPresent = implementor.GetTriggerStatus() == DataLoad.Triggers.TriggerStatus.Enabled; } catch (TriggerMissingException) { - present = false; + triggerPresent = false; } } } @@ -311,7 +310,7 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis archiveTable.AddColumn(column, new TypeGuesser.DatabaseTypeRequest(toProcess.Columns[column].DataType), true, 30000); } } - if (present) + if (triggerPresent) { string triggerProblems = ""; string triggerOK = ""; @@ -323,12 +322,12 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis existing = _destinationDatabase.ExpectTable(tblName); implementor = triggerFactory.Create(existing); - present = false; + triggerPresent = false; } } } - if (!present) + if (!triggerPresent) { implementor.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet); } diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs index 3fb6c20830..ccb6ab36e1 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs @@ -280,10 +280,7 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun += Environment.NewLine; - //var x = _archiveTable.DiscoverColumns(); - //var liveCols = _columns.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]").Union(new string[] var liveCols = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]") - //.Where(c => c != "[hic_validTo]" && c != "[hic_userID]" && c != "[hic_status]") .ToList(); if (!liveCols.Contains($"[{SpecialFieldNames.DataLoadRunID}]")) liveCols.Add($"[{SpecialFieldNames.DataLoadRunID}]"); @@ -291,10 +288,6 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch liveCols = liveCols.Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true).ToList(); var archiveCols = $"{string.Join(",", liveCols)}"; - //$",hic_validTo,hic_userID,hic_status"; - //if (!archiveCols.Contains("hic_validTo")) archiveCols += ",hic_validTo"; - //if (!archiveCols.Contains("hic_userID")) archiveCols += ",hic_userID "; - //if (!archiveCols.Contains("hic_status")) archiveCols += ",hic_status "; var cDotArchiveCols = string.Join(",", liveCols.Select(s => $"c.{s}")); @@ -323,8 +316,6 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun += $"\tAND{Environment.NewLine}"; //add an AND because there are more coming } - //TODO: ordering issue when adding a new column and removing an existing one - sqlToRun += string.Format("\tWHERE a.[{0}] IS NULL -- where archive record doesn't exist" + Environment.NewLine, _primaryKeys.First().GetRuntimeName()); sqlToRun += $"\tAND @index > ISNULL(c.{SpecialFieldNames.ValidFrom}, '1899/01/01'){Environment.NewLine}"; From c0c381e1b0da9d2770852bfa7e59f722758c3444 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 9 Jun 2026 10:53:11 +0100 Subject: [PATCH 6/8] update codeql --- .../Destinations/ExecuteFullExtractionToDatabaseMSSql.cs | 5 ++--- .../Implementations/MicrosoftSQLTriggerImplementer.cs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index aa7f94205b..f5bf12aba8 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -245,10 +245,9 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis cmd.ExecuteNonQuery(); var removedColumns = destinationColumns.Except(sourceColumns).Where(c => !SpecialFieldNames.IsHicPrefixed(c)); - foreach (var column in removedColumns) + foreach (var column in removedColumns.Select(c => existing.DiscoverColumn(c))) { - var discoveredColumn = existing.DiscoverColumn(column); - existing.DropColumn(discoveredColumn); + existing.DropColumn(column); } string triggerProblems = ""; string triggerOK = ""; diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs index ccb6ab36e1..584ff17506 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs @@ -285,7 +285,7 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch if (!liveCols.Contains($"[{SpecialFieldNames.DataLoadRunID}]")) liveCols.Add($"[{SpecialFieldNames.DataLoadRunID}]"); if (!liveCols.Contains($"[{SpecialFieldNames.ValidFrom}]")) liveCols.Add($"[{SpecialFieldNames.ValidFrom}]"); - liveCols = liveCols.Where(col => _dontAddDataLoadRunId ? col != $"[{SpecialFieldNames.DataLoadRunID}]" : true).ToList(); + liveCols = liveCols.Where(col => !_dontAddDataLoadRunId || col != $"[{SpecialFieldNames.DataLoadRunID}]").ToList(); var archiveCols = $"{string.Join(",", liveCols)}"; From 61810338c3c5659221bb4d2b08623585f4297d29 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 10 Jun 2026 08:52:10 +0100 Subject: [PATCH 7/8] add tests --- ...eMSSqlDestinationWithArchiveTriggerTest.cs | 1926 +++++++++++++++++ .../ExecuteFullExtractionToDatabaseMSSql.cs | 6 +- .../MicrosoftSQLTriggerImplementer.cs | 14 +- 3 files changed, 1937 insertions(+), 9 deletions(-) diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs index 34d50b3d06..cabae54af4 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs @@ -36,6 +36,7 @@ using Tests.Common; using Tests.Common.Scenarios; using TypeGuesser; +using YamlDotNet.Serialization.NodeDeserializers; namespace Rdmp.Core.Tests.DataExport.DataExtraction; @@ -774,4 +775,1929 @@ public void SQLServerDestinationWithTriggersNoPKs() { Assert.That(dt.Rows, Has.Count.EqualTo(2)); } + + //add a column + [Test] + public void SQLServerDestinationWithTriggersAddAColumn() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //remove a column + [Test] + public void SQLServerDestinationWithTriggersRemoveAColumn() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //remove a column, add a different column + [Test] + public void SQLServerDestinationWithTriggersRemoveAColumnAddAColumn() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + + + var cols_1 = ec.GetAllExtractableColumnsFor(eds); + var col_1 = cols_1.First(c => c.SelectSQL.Contains("current_address_L4")); + var order = col_1.Order; + var selectSQL = col_1.SelectSQL; + var cei = col_1.CatalogueExtractionInformation; + col_1.DeleteInDatabase(); + + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + var cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + //col.DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(38)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + //cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + //col.DeleteInDatabase(); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //remove a column, add the column back + [Test] + public void SQLServerDestinationWithTriggersRemoveAColumnAddItBack() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + + + var cols_1 = ec.GetAllExtractableColumnsFor(eds); + var col_1 = cols_1.First(c => c.SelectSQL.Contains("current_address_L4")); + var order = col_1.Order; + var selectSQL = col_1.SelectSQL; + var cei = col_1.CatalogueExtractionInformation; + //col_1.DeleteInDatabase(); + + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + var cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + //cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + //col.DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + + cols = ec.GetAllExtractableColumnsFor(eds); + //cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + //cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + //col.DeleteInDatabase(); + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //add a column, remove a different column + [Test] + public void SQLServerDestinationWithTriggersAddAColumnThenRemoveADifferentOne() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + //current_address_L4 + cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(3)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + //add a column, remove the same column + [Test] + public void SQLServerDestinationWithTriggersAddAColumnThenRemoveTheSameOne() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + + //current_address_L4 + cols = ec.GetAllExtractableColumnsFor(eds); + cols.First(c => c.SelectSQL.Contains("current_record")).DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(2)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } + + //add a column and remove a column + [Test] + public void SQLServerDestinationWithTriggersAddAColumnAndRemoveAtSameTime() + { + var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + //create catalogue from file + var csvFile = CreateFileInForLoading("bob.csv", 1, new Random(5000)); + // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) + var creator = new CataloguePipelinesAndReferencesCreation( + RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); + + // find the excel loading pipeline + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + + if (pipe is null) + { + creator.CreatePipelines(new PlatformDatabaseCreationOptions { }); + pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) + .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + } + + // run an import of the file using the pipeline + var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile( + new ThrowImmediatelyActivator(RepositoryLocator), + csvFile, + null, db, pipe, null); + + cmd.Execute(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); + var ei = chiColumnInfo.ExtractionInformation; + ei.IsExtractionIdentifier = true; + ei.IsPrimaryKey = true; + ei.SaveToDatabase(); + var project = new Project(DataExportRepository, "MyProject") + { + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() + }; + project.SaveToDatabase(); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + cic.CreateRootContainerIfNotExists(); + var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); + var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); + conf.SaveToDatabase(); + agg1.SaveToDatabase(); + cic.RootCohortAggregateContainer.AddChild(agg1, 0); + cic.SaveToDatabase(); + var dim = new AggregateDimension(CatalogueRepository, ei, agg1); + dim.SaveToDatabase(); + agg1.SaveToDatabase(); + + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; + const string ReleaseIdentifierFieldName = "ReleaseId"; + const string DefinitionTableForeignKeyField = "cohortDefinition_id"; + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + if (_cohortDatabase.Exists()) + DeleteTables(_cohortDatabase); + else + _cohortDatabase.Create(); + + var definitionTable = _cohortDatabase.CreateTable("CohortDefinition", new[] + { + new DatabaseColumnRequest("id", new DatabaseTypeRequest(typeof(int))) + { AllowNulls = false, IsAutoIncrement = true, IsPrimaryKey = true }, + new DatabaseColumnRequest("projectNumber", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("version", new DatabaseTypeRequest(typeof(int))) { AllowNulls = false }, + new DatabaseColumnRequest("description", new DatabaseTypeRequest(typeof(string), 3000)) + { AllowNulls = false }, + new DatabaseColumnRequest("dtCreated", new DatabaseTypeRequest(typeof(DateTime))) + { AllowNulls = false, Default = MandatoryScalarFunctions.GetTodaysDate } + }); + var idColumn = definitionTable.DiscoverColumn("id"); + var foreignKey = + new DatabaseColumnRequest(DefinitionTableForeignKeyField, new DatabaseTypeRequest(typeof(int)), false) + { IsPrimaryKey = true }; + + _cohortDatabase.CreateTable("Cohort", new[] + { + new DatabaseColumnRequest("chi", + new DatabaseTypeRequest(typeof(string)), false) + { + IsPrimaryKey = true, + + // if there is a single collation amongst private identifier prototype references we must use that collation + // when creating the private column so that the DBMS can link them no bother + Collation = null + }, + new DatabaseColumnRequest(ReleaseIdentifierFieldName, new DatabaseTypeRequest(typeof(string), 300)) + { AllowNulls = true }, + foreignKey + }); + + var newExternal = + new ExternalCohortTable(DataExportRepository, "TestExternalCohort", DatabaseType.MicrosoftSQLServer) + { + Database = CohortDatabaseName, + Server = _cohortDatabase.Server.Name, + DefinitionTableName = definitionTableName, + TableName = cohortTableName, + Name = ExternalCohortTableNameInCatalogue, + Username = _cohortDatabase.Server.ExplicitUsernameIfAny, + Password = _cohortDatabase.Server.ExplicitPasswordIfAny, + PrivateIdentifierField = "chi", + ReleaseIdentifierField = "ReleaseId", + DefinitionTableForeignKeyField = "cohortDefinition_id" + }; + + newExternal.SaveToDatabase(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); + var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( + new ThrowImmediatelyActivator(RepositoryLocator), + cic, + newExternal, + "MyCohort", + project, + cohortPipeline + ); + newCohortCmd.Execute(); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + + var ec = new ExtractionConfiguration(DataExportRepository, project) + { + Name = "ext1", + Cohort_ID = extractableCohort.ID + }; + var eds = new ExtractableDataSet(DataExportRepository, catalogue); + ec.AddDatasetToConfiguration(eds); + var cols = ec.GetAllExtractableColumnsFor(eds); + var col = cols.First(c => c.SelectSQL.Contains("current_record")); + var order = col.Order; + var selectSQL = col.SelectSQL; + var cei = col.CatalogueExtractionInformation; + col.DeleteInDatabase(); + cols = ec.GetAllExtractableColumnsFor(eds); + ec.SaveToDatabase(); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var component = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); + var destinationArguments = component.CreateArgumentsForClassIfNotExists() + .ToList(); + var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); + var argumentUseArchiveTrigger = destinationArguments.Single(a => a.Name == "UseArchiveTrigger"); + var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + { + Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, + Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, + Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny + }; + _extractionServer.SaveToDatabase(); + + argumentServer.SetValue(_extractionServer); + argumentServer.SaveToDatabase(); + argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); + argumentDbNamePattern.SaveToDatabase(); + argumentTblNamePattern.SetValue("$c_$d"); + argumentTblNamePattern.SaveToDatabase(); + argumentUseArchiveTrigger.SetValue(true); + argumentUseArchiveTrigger.SaveToDatabase(); + reExtract.SetValue(true); + reExtract.SaveToDatabase(); + + var component2 = new PipelineComponent(CatalogueRepository, extractionPipeline, + typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); + var arguments2 = component2.CreateArgumentsForClassIfNotExists() + .ToArray(); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); + arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + + //configure the component as the destination + extractionPipeline.DestinationPipelineComponent_ID = component.ID; + extractionPipeline.SourcePipelineComponent_ID = component2.ID; + extractionPipeline.SaveToDatabase(); + + + var dbname = TestDatabaseNames.GetConsistentName($"{project.Name}_{project.ProjectNumber}"); + var dbToExtractTo = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(dbname); + if (dbToExtractTo.Exists()) + dbToExtractTo.Drop(); + dbToExtractTo.Create(); + var runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + var returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + + + var destinationTable = dbToExtractTo.ExpectTable("ext1_bob"); + Assert.That(destinationTable.Exists()); + + var dt = destinationTable.GetDataTable(); + + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(39)); + var hicLoadID = dt.Rows[0].ItemArray[37]; + + var archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + var archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(0)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(42)); + ec.RemoveDatasetFromConfiguration(eds); + ec.AddDatasetToConfiguration(eds); + cols.First(c => c.SelectSQL.Contains("current_address_L4")).DeleteInDatabase(); + + runner = new ExtractionRunner(new ThrowImmediatelyActivator(RepositoryLocator), new ExtractionOptions + { + Command = CommandLineActivity.run, + ExtractionConfiguration = ec.ID.ToString(), + ExtractGlobals = true, + Pipeline = extractionPipeline.ID.ToString() + }); + + returnCode = runner.Run( + RepositoryLocator, + ThrowImmediatelyDataLoadEventListener.Quiet, + ThrowImmediatelyCheckNotifier.Quiet, + new GracefulCancellationToken()); + + Assert.That(returnCode, Is.EqualTo(0), "Return code from runner was non zero"); + + Assert.That(destinationTable.Exists()); + + dt = destinationTable.GetDataTable(); + Assert.That(dt.Rows, Has.Count.EqualTo(1)); + Assert.That(dt.Columns, Has.Count.EqualTo(40)); + + archiveTable = dbToExtractTo.ExpectTable("ext1_bob_Archive"); + Assert.That(archiveTable.Exists()); + archive_dt = archiveTable.GetDataTable(); + Assert.That(archive_dt.Rows, Has.Count.EqualTo(1)); + Assert.That(archive_dt.Columns, Has.Count.EqualTo(43)); + } } \ No newline at end of file diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index f5bf12aba8..3c50006cf4 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -4,9 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using Amazon.Auth.AccessControlPolicy; using FAnsi.Discovery; -using NPOI.SS.Formula.Functions; using Rdmp.Core.CommandExecution; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataExport.Data; @@ -15,7 +13,6 @@ using Rdmp.Core.DataExport.DataRelease.Pipeline; using Rdmp.Core.DataExport.DataRelease.Potential; using Rdmp.Core.DataFlowPipeline; -using Rdmp.Core.DataLoad.Engine.Job.Scheduling; using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; using Rdmp.Core.DataLoad.Triggers; using Rdmp.Core.DataLoad.Triggers.Exceptions; @@ -33,7 +30,6 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; -using YamlDotNet.Core; namespace Rdmp.Core.DataExport.DataExtraction.Pipeline.Destinations; @@ -227,6 +223,8 @@ private DataTableUploadDestination PrepareDestination(IDataLoadEventListener lis Destination PKs: {string.Join(", ", remotePKs)} """)); return null;//todo this error could be better + //todo: need to make the same changes on the merge component + //todo: figure out the tests } if (hasStructuralChanges(toProcess, existing)) { diff --git a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs index 584ff17506..1163ea2711 100644 --- a/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs +++ b/Rdmp.Core/DataLoad/Triggers/Implementations/MicrosoftSQLTriggerImplementer.cs @@ -261,16 +261,19 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch sqlToRun = sqlToRun.Trim(); if (!sqlToRun.Contains("hic_validTo")) { - sqlToRun += $"{Environment.NewLine}"; - sqlToRun += $",\thic_validTo datetime"; + sqlToRun += $",{Environment.NewLine}"; + sqlToRun += $"hic_validTo datetime"; } if (!sqlToRun.Contains("hic_userID")) { - sqlToRun += ",\thic_userID varchar(128)"; + sqlToRun += $",{Environment.NewLine}"; + + sqlToRun += "hic_userID varchar(128)"; } if (!sqlToRun.Contains("hic_status")) { - sqlToRun += ",\thic_status char(1)"; + sqlToRun += $",{Environment.NewLine}"; + sqlToRun += "hic_status char(1)"; } @@ -298,12 +301,13 @@ private void CreateViewOldVersionsTableValuedFunction(string sqlUsedToCreateArch ", '1899/01/01') AND hic_validTo" + Environment.NewLine, _archiveTable); sqlToRun += Environment.NewLine; + //these are columns that exist in the archive, but not the table. So have to be set as NULL as var nullCoulmns = _archiveTable.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]").Except(_table.DiscoverColumns().Select(c => $"[{c.GetRuntimeName()}]")); cDotArchiveCols = string.Join(",", liveCols.Select(s => nullCoulmns.Contains(s) ? $"NULL AS {s}" : $"c.{s}")); sqlToRun += $"\tINSERT @returntable{Environment.NewLine}"; sqlToRun += - $"\tSELECT {cDotArchiveCols}";//,NULL AS hic_validTo, NULL AS hic_userID, 'C' AS hic_status{Environment.NewLine}"; //c is for current + $"\tSELECT {cDotArchiveCols}"; sqlToRun += string.Format("\tFROM [{0}] c" + Environment.NewLine, _table.GetRuntimeName()); sqlToRun += $"\tLEFT OUTER JOIN @returntable a ON {Environment.NewLine}"; From f7cb8b8ac39c68aee15a1ee58fdb773df5a6b42c Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 10 Jun 2026 10:44:59 +0100 Subject: [PATCH 8/8] update test pipelines --- ...tabaseMSSqlDestinationWithArchiveTriggerTest.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs index cabae54af4..e0eb24dac4 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationWithArchiveTriggerTest.cs @@ -917,7 +917,7 @@ public void SQLServerDestinationWithTriggersAddAColumn() col.DeleteInDatabase(); cols = ec.GetAllExtractableColumnsFor(eds); ec.SaveToDatabase(); - var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 4"); var component = new PipelineComponent(CatalogueRepository, extractionPipeline, typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); var destinationArguments = component.CreateArgumentsForClassIfNotExists() @@ -1161,7 +1161,7 @@ public void SQLServerDestinationWithTriggersRemoveAColumn() var eds = new ExtractableDataSet(DataExportRepository, catalogue); ec.AddDatasetToConfiguration(eds); ec.SaveToDatabase(); - var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 5"); var component = new PipelineComponent(CatalogueRepository, extractionPipeline, typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); var destinationArguments = component.CreateArgumentsForClassIfNotExists() @@ -1414,7 +1414,7 @@ public void SQLServerDestinationWithTriggersRemoveAColumnAddAColumn() var eds = new ExtractableDataSet(DataExportRepository, catalogue); ec.AddDatasetToConfiguration(eds); ec.SaveToDatabase(); - var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 6"); var component = new PipelineComponent(CatalogueRepository, extractionPipeline, typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); var destinationArguments = component.CreateArgumentsForClassIfNotExists() @@ -1710,7 +1710,7 @@ public void SQLServerDestinationWithTriggersRemoveAColumnAddItBack() var eds = new ExtractableDataSet(DataExportRepository, catalogue); ec.AddDatasetToConfiguration(eds); ec.SaveToDatabase(); - var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 7"); var component = new PipelineComponent(CatalogueRepository, extractionPipeline, typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); var destinationArguments = component.CreateArgumentsForClassIfNotExists() @@ -2013,7 +2013,7 @@ public void SQLServerDestinationWithTriggersAddAColumnThenRemoveADifferentOne() col.DeleteInDatabase(); cols = ec.GetAllExtractableColumnsFor(eds); ec.SaveToDatabase(); - var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 8"); var component = new PipelineComponent(CatalogueRepository, extractionPipeline, typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); var destinationArguments = component.CreateArgumentsForClassIfNotExists() @@ -2300,7 +2300,7 @@ public void SQLServerDestinationWithTriggersAddAColumnThenRemoveTheSameOne() col.DeleteInDatabase(); cols = ec.GetAllExtractableColumnsFor(eds); ec.SaveToDatabase(); - var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 9"); var component = new PipelineComponent(CatalogueRepository, extractionPipeline, typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); var destinationArguments = component.CreateArgumentsForClassIfNotExists() @@ -2588,7 +2588,7 @@ public void SQLServerDestinationWithTriggersAddAColumnAndRemoveAtSameTime() col.DeleteInDatabase(); cols = ec.GetAllExtractableColumnsFor(eds); ec.SaveToDatabase(); - var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 3"); + var extractionPipeline = new Pipeline(CatalogueRepository, "Empty extraction pipeline 10"); var component = new PipelineComponent(CatalogueRepository, extractionPipeline, typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); var destinationArguments = component.CreateArgumentsForClassIfNotExists()