From 914df339d1b850ac555f3286db4299e6534702c8 Mon Sep 17 00:00:00 2001 From: Thierry Schork Date: Mon, 14 Apr 2025 20:59:24 +0200 Subject: [PATCH] added powershell jobs deployed on prodDB to execute sync and production --- HCI - BAG symmetricDS/proddb to mi backup.ps1 | 453 ++++++++++++++++++ HCI - BAG symmetricDS/sl2007 production.ps1 | 118 +++++ .../start snapshots sync.ps1 | 115 +++++ 3 files changed, 686 insertions(+) create mode 100644 HCI - BAG symmetricDS/proddb to mi backup.ps1 create mode 100644 HCI - BAG symmetricDS/sl2007 production.ps1 create mode 100644 HCI - BAG symmetricDS/start snapshots sync.ps1 diff --git a/HCI - BAG symmetricDS/proddb to mi backup.ps1 b/HCI - BAG symmetricDS/proddb to mi backup.ps1 new file mode 100644 index 0000000..ae08bc9 --- /dev/null +++ b/HCI - BAG symmetricDS/proddb to mi backup.ps1 @@ -0,0 +1,453 @@ +#NOSQLPS + +function log-out{ + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string] $msg + ) + Write-Output "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") :: $msg" +} + +<# +.SYNOPSIS +Push a superset from prodDB to the BAG Azure managed sql instance + +.DESCRIPTION +This function will do those operations: +- Check if the superset given is a snapshot or a database +if it's a snapshot, the source database behind the snapshot is backed up +- Authenticate to Azur using a service principal (with a certificate authentication) +- Create a SAS tocken from Azure Blob Storage +- Refresh credentials on prodDB to access the storage account +- Check if a blob of the superset we want to push is more than 1 day old + - if no blob exists, force a backup + - if a blob exists but is aged of less than 1 full day, skip the backup + - if a blob exists but is older than 1 full day, force a new backup +- Refresh the credential with the SAS token on the BAG managed sql instance +- Drop the target database if it exists (we cannot restore over an existing db in managed instances) +- Restore the backup in the cloud db +- Create logins on the restored db for the login [sql-au_bag_apv] +- Give db_datareader and EXECUTE permission on the restored db to [sql-au_bag_apv] + +.PARAMETERS +[string] $supersetToCopy +The name of the superset to transfert. for exemple: product_superset + +$backupBehavior +auto = check age of blob and skip if backup is less than 1 day old, $true to force backup, $false to skip backup. + +.EXAMPLES +push-superset -supersetToCopy Artikel_History_Superset + +#> +function push-superset{ + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string] $supersetToCopy, + + [Parameter(mandatory=$false)] + [string] $backupBehavior = "Auto" + ) + + # Variables + $tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica + $clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup + $skipBackup = $false + + $maxRetries = 5 + $retryInterval = 10 # seconds + $retryCount = 0 + $success = $false + + + log-out -msg "----------------------------------------------" + log-out -msg "---- Starting push of $supersetToCopy to the cloud" + log-out -msg "----------------------------------------------" + + ##do not alter below + $serverInstance = "SWMDATASQLPRD01.centralinfra.net" + $databaseName = $null + $storageAccountName = "stbagspezlisteprdsql" + $containerName = "sqlbakfiles" + $resourceGroupName="rg-BAGSpezListePrd-SQL" + $vaultName="kv-BAGSpezListePrd-bkp" + $secretName="superuser" + $blobPermissions="rwd" + $expiryTime = (Get-Date).AddHours(1) # SAS token valid for 1 hour + $backupFileName = "$supersetToCopy.bak" + + <# + $mod = Get-Module -Name Az.Accounts + if($null -eq $mod){ + Install-Module -Name Az.KeyVault -Scope CurrentUser -force + } + + $mod = Get-Module -Name Az.Storage + if($null -eq $mod){ + Install-Module -Name Az.Storage -Scope CurrentUser -Force + } + + $mod = Get-Module -Name sqlserver + if($null -eq $mod){ + Install-Module -Name sqlserver -Scope CurrentUser -AllowClobber -force + } + #> + + + #Managed instance related + $MILogin='superuser' + $MIPwd=$null #fetched from key vault later + $MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net" + + #check for the origin of product_superset + $query=" + SELECT s.name AS superset, d.name AS srcDb + FROM sys.databases s + JOIN sys.databases d ON d.[database_id] = s.[source_database_id] + WHERE s.name='$supersetToCopy' + " + $res = Invoke-Sqlcmd -ServerInstance $serverInstance -Query $query + $databaseName = $res.srcDb + if($null -eq $databaseName){ + #superset is not a db snapshot... + $databaseName = $supersetToCopy + } + + # Log in to Azure + $retryCount = 0 + $success = $false + while (-not $success -and $retryCount -lt $maxRetries) { + try{ + $null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } + } + + if(-not $success){ + write-error "Error authenticating to azure" + exit 1 + } + + # Get the storage account context + $storageAccount = Get-AzStorageAccount -ResourceGroupName $resourceGroupName -Name $storageAccountName + $context = $storageAccount.Context + + # Get the blob properties + $backupFile = "https://$storageAccountName.blob.core.windows.net/$containerName/$backupFileName" + if($backupAutoBehavior -eq "Auto") + { + try{ + $blob = Get-AzStorageBlob -Container $containerName -Blob $backupFileName -Context $context + } + catch{ + $blob = $null + } + + if($null -eq $blob){ + $skipBackup=$false + log-out -msg "Blob does not exists, forcing backup" + } + else{ + # Calculate the age of the blob + $currentDate = Get-Date + $blobLastModified = $blob | Select-Object -ExpandProperty LastModified + $blobLastModifiedDateTime = [DateTime]::Parse($blobLastModified.ToString()) + $blobAge = $currentDate - $blobLastModifiedDateTime + + if($blobAge.Days -eq 0){ + $skipBackup=$true + log-out -msg "Blob exists and is less than a day old, skipping backup" + } + else{ + $skipBackup=$false + log-out -msg "Blob exists and is older than a day, forcing backup" + } + } + } + else{ + if($backupAutoBehavior -eq "Force"){ + $skipBackup = $false + log-out -msg "Forcing backup (param)" + } + + if($backupAutoBehavior -eq "Skip"){ + $skipBackup = $true + log-out -msg "Skipping backup (param)" + } + else{ + $skipBackup = $false + log-out -msg "Forcing backup (empty param)" + + } + } + + # Generate the SAS token + $sasToken = New-AzStorageBlobSASToken -Context $context -Container $containerName -Blob $backupFileName -Permission $blobPermissions -ExpiryTime $expiryTime + + # Save token in db + $sqlQuery = " +IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName') +BEGIN + CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] + WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; +END +ELSE +BEGIN + ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] + WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; +END + " + $retryCount = 0 + $success = $false + + while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery + log-out -msg "Credential refreshed on $serverInstance" + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } + } + + if(-not $success){ + write-error "Error refreshing credentials on $serverInstance" + exit 1 + } + + #Back Up Database + $url=$backupFile + $sqlQuery = " +BACKUP DATABASE [$databaseName] +TO URL = N'$url' +WITH FORMAT, MEDIANAME = 'SQLServerBackups', NAME = 'Full Backup of $databaseName'; + " + if($false -eq $skipBackup){ + $retryCount = 0 + $success = $false + + while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery + log-out -msg "backup done" + $success = $true + } + catch{ + $retryCount ++ + wait-Start-Sleep -Seconds $retryInterval + } + } + + if(-not $success){ + write-error "Error backing up $databaseName" + exit 1 + } + } + else{ + log-out -msg "Backup skipped." + } + + #fetch the Managed Instance password + $retryCount = 0 + $success = $false + while (-not $success -and $retryCount -lt $maxRetries) { + try{ + $MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText + $success = $true + } + catch{ + $retryCount ++ + Start-Sleep -Seconds $retryInterval + } + } + if(-not $success){ + write-error "Error getting secret from key vault" + exit 1 + } + + #craft credential for MI connection + $securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force + $credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword + + #add sas token + $sqlCred=" +IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName') +BEGIN + CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] + WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; +END +ELSE +BEGIN + ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] + WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; +END + " + $retryCount = 0 + $success = $false + + while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlCred -Credential $credentialMI + log-out -msg "Credential refreshed on $MIInstance" + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } + } + + if(-not $success){ + write-error "Error refreshing credentials on $MIInstance" + exit 1 + } + + + #drop existing db + $sqlDrop=" + +IF EXISTS( + SELECT 1 + FROM sys.databases d + WHERE d.name ='$supersetToCopy' +) +BEGIN + DROP DATABASE $supersetToCopy; +END + " + $retryCount = 0 + $success = $false + while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlDrop -Credential $credentialMI + log-out -msg "Dropped existing $supersetToCopy db (if needed)" + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } + } + + if(-not $success){ + write-error "Error dropping existing db $supersetToCopy" + exit 1 + } + + + #restore superset + $url=$backupFile + $sqlRestore="RESTORE DATABASE [$supersetToCopy] FROM URL = N'$url'; " + $retryCount = 0 + $success = $false + while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlRestore -Credential $credentialMI + log-out -msg "Restored $supersetToCopy" + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } + } + + if(-not $success){ + write-error "Error restoring db $supersetToCopy" + exit 1 + } + + + + #create user for sql-au_bag_apv + $sqlUser=" + +IF NOT EXISTS ( + SELECT * + FROM sys.database_principals s + WHERE s.name='sql-au_bag_apv' +) +BEGIN + CREATE USER [sql-au_bag_apv] FOR LOGIN [sql-au_bag_apv]; +END + " + $retryCount = 0 + $success = $false + while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlUser -Credential $credentialMI -Database $supersetToCopy + log-out -msg "Created user for login [sql-au_bag_apv] in $supersetToCopy" + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } + } + + if(-not $success){ + write-error "Error creating user in $supersetToCopy" + exit 1 + } + + + #give db_datareader + $sqlPerms=" + +IF '$supersetToCopy' = 'sl2007' +BEGIN + ALTER ROLE [db_owner] ADD MEMBER [sql-au_bag_apv]; +END +ELSE +BEGIN + ALTER ROLE [db_datareader] ADD MEMBER [sql-au_bag_apv]; + ALTER ROLE [db_datawriter] ADD MEMBER [sql-au_bag_apv]; +END +GRANT EXECUTE TO [sql-au_bag_apv]; + + " + $retryCount = 0 + $success = $false + while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlPerms -Credential $credentialMI -Database $supersetToCopy + log-out -msg "Added user [sql-au_bag_apv] with read and execute permissions to $supersetToCopy" + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } + } + + if(-not $success){ + write-error "Error giving permissions to user in $supersetToCopy" + exit 1 + } + +} + +log-out -msg "Starting transfert of Artikel_History_Superset" +push-superset -supersetToCopy Artikel_History_Superset -backupBehavior "Force" +log-out -msg "Done" + +log-out -msg "Starting transfert of product_Superset" +push-superset -supersetToCopy product_superset -backupBehavior "Force" +log-out -msg "Done" + +log-out -msg "Starting transfert of artikel_superset" +push-superset -supersetToCopy artikel_superset -backupBehavior "Force" +log-out -msg "Done" + +log-out -msg "Starting transfert of artikel_superset" +push-superset -supersetToCopy ODB_JobManager -backupBehavior "Force" +log-out -msg "Done" + \ No newline at end of file diff --git a/HCI - BAG symmetricDS/sl2007 production.ps1 b/HCI - BAG symmetricDS/sl2007 production.ps1 new file mode 100644 index 0000000..accde60 --- /dev/null +++ b/HCI - BAG symmetricDS/sl2007 production.ps1 @@ -0,0 +1,118 @@ +# Variables +$tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica +$clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup +$vaultName="kv-BAGSpezListePrd-bkp" +$secretName="superuser" + +$maxRetries = 5 +$retryInterval = 10 # seconds +$retryCount = 0 +$success = $false + +#Managed instance related +$MILogin='superuser' +$MIPwd=$null #fetched from key vault later +$MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net" + +function log-out{ + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string] $msg + ) + Write-Output "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") :: $msg" +} + +# Log in to Azure +$retryCount = 0 +$success = $false + +while (-not $success -and $retryCount -lt $maxRetries) { + try{ + $null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } +} + +if(-not $success){ + write-error "Error auhthenticating to azure" + exit 1 +} + +#fetch the Managed Instance password +$retryCount = 0 +$success = $false + +while (-not $success -and $retryCount -lt $maxRetries) { + try{ + $MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } +} + +if(-not $success){ + write-error "Error fetching password from key vault" + exit 1 +} + +#craft credential for MI connection +$securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force +$credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword + +#start sl2007.daily_batch_update +log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") starting sl2007 daily update" +$query=" +exec sl2007.dbo.usp_Daily_Batch_Update +" +$retryCount = 0 +$success = $false + +while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $query -Credential $credentialMI + log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") sl2007 daily update done" + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } +} + +if(-not $success){ + write-error "Error starting usp_Daily_Batch_Update" + exit 1 +} + +#start sl2007_superset generation +log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") starting sl2007_superset generation" +$query=" +exec sl2007_superset.dbo.p_Fill__BAG__SL2007_Superset +" +$retryCount = 0 +$success = $false + +while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $query -Credential $credentialMI + log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") sl2007_superset generation done" + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } +} + +if(-not $success){ + write-error "Error starting sl2007_suèerset production" + exit 1 +} diff --git a/HCI - BAG symmetricDS/start snapshots sync.ps1 b/HCI - BAG symmetricDS/start snapshots sync.ps1 new file mode 100644 index 0000000..139402d --- /dev/null +++ b/HCI - BAG symmetricDS/start snapshots sync.ps1 @@ -0,0 +1,115 @@ +# Variables +$tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica +$clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup +$vaultName="kv-BAGSpezListePrd-bkp" +$secretName="superuser" + +$maxRetries = 5 +$retryInterval = 10 # seconds +$retryCount = 0 +$success = $false + +#Managed instance related +$MILogin='superuser' +$MIPwd=$null #fetched from key vault later +$MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net" + +function log-out{ + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string] $msg + ) + Write-Output "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") :: $msg" +} + +while (-not $success -and $retryCount -lt $maxRetries) { + try{ + $null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d + $success = $true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } +} + +if(-not $success){ + write-error "Error auhthenticating to azure" + exit 1 +} + +#fetch the Managed Instance password +$retryCount = 0 +$success = $false + +while (-not $success -and $retryCount -lt $maxRetries) { + try{ + $MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText + $success = $true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } +} + +if(-not $success){ + write-error "Error fetching password from key vault" + exit 1 +} + +#craft credential for MI connection +$securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force +$credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword + +#start sl2007.daily_batch_update +log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") starting sl2007 snapshot" +$query=" +exec dba.dbo.start_snapshot @published_db='sl2007', @debug=0 +" +$retryCount = 0 +$success = $false + +while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $query -Credential $credentialMI + log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") sl2007 snapshot done" + $success = $true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } +} + +if(-not $success){ + write-error "Error generating sl2007 snapshot" + exit 1 +} + + +#start sl2007_superset generation +log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") starting sl2007_superset snapshot" +$query=" +exec dba.dbo.start_snapshot @published_db='sl2007_superset', @debug=0 +" +$retryCount = 0 +$success = $false + +while (-not $success -and $retryCount -lt $maxRetries) { + try{ + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $query -Credential $credentialMI + log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") sl2007_superset snapshot done" + $success=$true + } + catch{ + $retryCount++ + Start-Sleep -Seconds $retryInterval + } +} + +if(-not $success){ + write-error "Error generating sl2007_superset snapshot" + exit 1 +}