From f478d2adeae9d949d6c799db9865d91c31713526 Mon Sep 17 00:00:00 2001 From: Thierry Schork Date: Wed, 19 Mar 2025 16:16:12 +0100 Subject: [PATCH] transformed to a function --- HCI - BAG symmetricDS/hci_bag_backup.ps1 | 297 +++++++++++++---------- 1 file changed, 167 insertions(+), 130 deletions(-) diff --git a/HCI - BAG symmetricDS/hci_bag_backup.ps1 b/HCI - BAG symmetricDS/hci_bag_backup.ps1 index e521c1e..e6e4ba2 100644 --- a/HCI - BAG symmetricDS/hci_bag_backup.ps1 +++ b/HCI - BAG symmetricDS/hci_bag_backup.ps1 @@ -1,107 +1,133 @@ -# Variables -$tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica -$clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup -$supersetToCopy="product_superset" -$supersetToCopy="Artikel_History_Superset" -$skipBackup = "auto" #auto = check age and skip if backup is less than 1 day old, $true to force skip backup, $false to force backup - <# -backup product_superset__0 : 00:13:00 -restore product_superset : 00:05:40 +.SYNOPSIS +Push a superset from prodDB to the BAG Azure managed sql instance + +.DESCRIPTION +This function will do those operations: +- Check if the superset given is a snapshot or a database + if it's a snapshot, the source database behind the snapshot is backed up +- Authenticate to Azur using a service principal (with a certificate authentication) +- Create a SAS tocken from Azure Blob Storage +- Refresh credentials on prodDB to access the storage account +- Check if a blob of the superset we want to push is more than 1 day old + - if no blob exists, force a backup + - if a blob exists but is aged of less than 1 full day, skip the backup + - if a blob exists but is older than 1 full day, force a new backup +- Refresh the credential with the SAS token on the BAG managed sql instance +- Drop the target database if it exists (we cannot restore over an existing db in managed instances) +- Restore the backup in the cloud db +- Create logins on the restored db for the login [sql-au_bag_apv] +- Give db_datareader and EXECUTE permission on the restored db to [sql-au_bag_apv] + +.PARAMETERS +[string] $supersetToCopy +The name of the superset to transfert. for exemple: product_superset + +.EXAMPLES +push-superset -supersetToCopy Artikel_History_Superset + #> +function push-superset([string] $supersetToCopy){ + + # Variables + $tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica + $clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup + $skipBackup = "auto" #auto = check age and skip if backup is less than 1 day old, $true to force skip backup, $false to force backup -$mod = Get-Module -Name Az.Accounts -if($null -eq $mod){ - Install-Module -Name Az.KeyVault -Scope CurrentUser -} + write-info "Starting push of $supersetToCopy to the cloud" -$mod = Get-Module -Name Az.Storage -if($null -eq $mod){ - Install-Module -Name Az.Storage -Scope CurrentUser -} + ##do not alter below + $serverInstance = "SWMDATASQLPRD01.centralinfra.net" + $databaseName = $null + $storageAccountName = "stbagspezlisteprdsql" + $containerName = "sqlbakfiles" + $resourceGroupName="rg-BAGSpezListePrd-SQL" + $vaultName="kv-BAGSpezListePrd-bkp" + $secretName="superuser" + $blobPermissions="rwd" + $expiryTime = (Get-Date).AddHours(1) # SAS token valid for 1 hour + $backupFileName = "$supersetToCopy.bak" -$mod = Get-Module -Name sqlserver -if($null -eq $mod){ - Install-Module -Name sqlserver -Scope CurrentUser -AllowClobber -} - - -##do not alter below -$serverInstance = "SWMDATASQLPRD01.centralinfra.net" -$databaseName = $null -$storageAccountName = "stbagspezlisteprdsql" -$containerName = "sqlbakfiles" -$resourceGroupName="rg-BAGSpezListePrd-SQL" -$vaultName="kv-BAGSpezListePrd-bkp" -$secretName="superuser" -$blobPermissions="rwd" -$expiryTime = (Get-Date).AddHours(1) # SAS token valid for 1 hour -$backupFileName = "$supersetToCopy.bak" - -#Managed instance related -$MILogin='superuser' -$MIPwd=$null #fetched from key vault later -$MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net" - -#check for the origin of product_superset -$query=" -SELECT s.name AS superset, d.name AS srcDb -FROM sys.databases s - JOIN sys.databases d ON d.[database_id] = s.[source_database_id] -WHERE s.name='$supersetToCopy' -" -$res = Invoke-Sqlcmd -ServerInstance $serverInstance -Query $query -$databaseName = $res.srcDb -if($null -eq $databaseName){ - #superset is not a db snapshot... - $databaseName = $supersetToCopy -} - -# Log in to Azure -$null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d - -# Get the storage account context -$storageAccount = Get-AzStorageAccount -ResourceGroupName $resourceGroupName -Name $storageAccountName -$context = $storageAccount.Context - -# Get the blob properties -$backupFile = "https://$storageAccountName.blob.core.windows.net/$containerName/$backupFileName" -if($skipBackup -eq "auto") -{ - try{ - $blob = Get-AzStorageBlob -Container $containerName -Blob $backupFileName -Context $context - } - catch{ - $blob = $null + $mod = Get-Module -Name Az.Accounts + if($null -eq $mod){ + Install-Module -Name Az.KeyVault -Scope CurrentUser } - if($null -eq $blob){ - $skipBackup=$false - Write-Output "Blob does not exists, forcing backup" + $mod = Get-Module -Name Az.Storage + if($null -eq $mod){ + Install-Module -Name Az.Storage -Scope CurrentUser } - else{ - # Calculate the age of the blob - $currentDate = Get-Date - $blobLastModified = $blob | Select-Object -ExpandProperty LastModified - $blobLastModifiedDateTime = [DateTime]::Parse($blobLastModified.ToString()) - $blobAge = $currentDate - $blobLastModifiedDateTime - if($blobAge.Days -eq 0){ - $skipBackup=$true - Write-Output "Blob exists and is less than a day old, skipping backup" + $mod = Get-Module -Name sqlserver + if($null -eq $mod){ + Install-Module -Name sqlserver -Scope CurrentUser -AllowClobber + } + + + #Managed instance related + $MILogin='superuser' + $MIPwd=$null #fetched from key vault later + $MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net" + + #check for the origin of product_superset + $query=" + SELECT s.name AS superset, d.name AS srcDb + FROM sys.databases s + JOIN sys.databases d ON d.[database_id] = s.[source_database_id] + WHERE s.name='$supersetToCopy' + " + $res = Invoke-Sqlcmd -ServerInstance $serverInstance -Query $query + $databaseName = $res.srcDb + if($null -eq $databaseName){ + #superset is not a db snapshot... + $databaseName = $supersetToCopy + } + + # Log in to Azure + $null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d + + # Get the storage account context + $storageAccount = Get-AzStorageAccount -ResourceGroupName $resourceGroupName -Name $storageAccountName + $context = $storageAccount.Context + + # Get the blob properties + $backupFile = "https://$storageAccountName.blob.core.windows.net/$containerName/$backupFileName" + if($skipBackup -eq "auto") + { + try{ + $blob = Get-AzStorageBlob -Container $containerName -Blob $backupFileName -Context $context + } + catch{ + $blob = $null + } + + if($null -eq $blob){ + $skipBackup=$false + Write-Output "Blob does not exists, forcing backup" } else{ - $skipBackup=$false - Write-Output "Blob exists and is older than a day, forcing backup" + # Calculate the age of the blob + $currentDate = Get-Date + $blobLastModified = $blob | Select-Object -ExpandProperty LastModified + $blobLastModifiedDateTime = [DateTime]::Parse($blobLastModified.ToString()) + $blobAge = $currentDate - $blobLastModifiedDateTime + + if($blobAge.Days -eq 0){ + $skipBackup=$true + Write-Output "Blob exists and is less than a day old, skipping backup" + } + else{ + $skipBackup=$false + Write-Output "Blob exists and is older than a day, forcing backup" + } } } -} -# Generate the SAS token -$sasToken = New-AzStorageBlobSASToken -Context $context -Container $containerName -Blob $backupFileName -Permission $blobPermissions -ExpiryTime $expiryTime + # Generate the SAS token + $sasToken = New-AzStorageBlobSASToken -Context $context -Container $containerName -Blob $backupFileName -Permission $blobPermissions -ExpiryTime $expiryTime -# Save token in db -$sqlQuery = " + # Save token in db + $sqlQuery = " IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName') BEGIN CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] @@ -112,34 +138,34 @@ BEGIN ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; END -" -Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery -Write-Output "Credential refreshed on $serverInstance" + " + Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery + Write-Output "Credential refreshed on $serverInstance" -#Back Up Database -$url=$backupFile -$sqlQuery = " + #Back Up Database + $url=$backupFile + $sqlQuery = " BACKUP DATABASE [$databaseName] TO URL = N'$url' WITH FORMAT, MEDIANAME = 'SQLServerBackups', NAME = 'Full Backup of $databaseName'; -" -if($false -eq $skipBackup){ - Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery - Write-Output "backup done" -} -else{ - write-output "Backup skipped." -} + " + if($false -eq $skipBackup){ + Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery + Write-Output "backup done" + } + else{ + write-output "Backup skipped." + } -#fetch the Managed Instance password -$MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText + #fetch the Managed Instance password + $MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText -#craft credential for MI connection -$securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force -$credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword + #craft credential for MI connection + $securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force + $credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword -#add sas token -$sqlCred=" + #add sas token + $sqlCred=" IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName') BEGIN CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] @@ -150,12 +176,12 @@ BEGIN ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; END -" -Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlCred -Credential $credentialMI -Write-Output "Credential refreshed on $MIInstance" + " + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlCred -Credential $credentialMI + Write-Output "Credential refreshed on $MIInstance" -#drop existing db -$sqlDrop=" + #drop existing db + $sqlDrop=" IF EXISTS( SELECT 1 @@ -165,20 +191,20 @@ IF EXISTS( BEGIN DROP DATABASE $supersetToCopy; END -" -Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlDrop -Credential $credentialMI -Write-Output "Dropped existing $supersetToCopy db (if needed)" + " + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlDrop -Credential $credentialMI + Write-Output "Dropped existing $supersetToCopy db (if needed)" -#restore superset -$url=$backupFile + #restore superset + $url=$backupFile $sqlRestore=" RESTORE DATABASE [$supersetToCopy] FROM URL = N'$url' " -Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlRestore -Credential $credentialMI -write-output "Restored $supersetToCopy" + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlRestore -Credential $credentialMI + write-output "Restored $supersetToCopy" -#create user for sql-au_bag_apv -$sqlUser=" + #create user for sql-au_bag_apv + $sqlUser=" IF NOT EXISTS ( SELECT * @@ -189,16 +215,27 @@ BEGIN CREATE USER [sql-au_bag_apv] FOR LOGIN [sql-au_bag_apv]; END " -Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlUser -Credential $credentialMI -Database $supersetToCopy -write-output "Created user for login [sql-au_bag_apv] in $supersetToCopy" + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlUser -Credential $credentialMI -Database $supersetToCopy + write-output "Created user for login [sql-au_bag_apv] in $supersetToCopy" -#give db_datareader -$sqlPerms=" + #give db_datareader + $sqlPerms=" ALTER ROLE [db_datareader] ADD MEMBER [sql-au_bag_apv]; GRANT EXECUTE TO [sql-au_bag_apv]; " -Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlPerms -Credential $credentialMI -Database $supersetToCopy -write-output "Added user [sql-au_bag_apv] with read and execute permissions to $supersetToCopy" + Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlPerms -Credential $credentialMI -Database $supersetToCopy + write-output "Added user [sql-au_bag_apv] with read and execute permissions to $supersetToCopy" + +} + +<# +backup product_superset__0 : 00:13:00 +restore product_superset : 00:05:40 +#> + +#push-superset -supersetToCopy sl2007_superset +push-superset -supersetToCopy Artikel_History_Superset +#push-superset -supersetToCopy product_superset \ No newline at end of file