#NOSQLPS function log-out{ [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string] $msg ) Write-Output "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") :: $msg" } <#1 .SYNOPSIS Check if a db with the specified name exists on the managed instant .PARAMETER [string] $superset The name of the database to check if it exists on the managed instance #> function find-db-exists-MI{ [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string] $superset ) $ret = $false $sql_check=" select 1 as ret from sys.databases d where name = '$superset' " $check_result = Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sql_check -Credential $credentialMI if($check_result -and $check_result["ret"] -eq "1"){ $ret = $true } return $ret } <# .SYNOPSIS Push a superset from prodDB to the BAG Azure managed sql instance .DESCRIPTION This function will do those operations: - Check if the superset given is a snapshot or a database if it's a snapshot, the source database behind the snapshot is backed up - Authenticate to Azur using a service principal (with a certificate authentication) - Create a SAS tocken from Azure Blob Storage - Refresh credentials on prodDB to access the storage account - Check if a blob of the superset we want to push is more than 1 day old - if no blob exists, force a backup - if a blob exists but is aged of less than 1 full day, skip the backup - if a blob exists but is older than 1 full day, force a new backup - Refresh the credential with the SAS token on the BAG managed sql instance - Drop the target database if it exists (we cannot restore over an existing db in managed instances) - Restore the backup in the cloud db - Create logins on the restored db for the login [sql-au_bag_apv] - Give db_datareader and EXECUTE permission on the restored db to [sql-au_bag_apv] .PARAMETERS [string] $supersetToCopy The name of the superset to transfert. for exemple: product_superset $backupBehavior auto = check age of blob and skip if backup is less than 1 day old, $true to force backup, $false to skip backup. .EXAMPLES push-superset -supersetToCopy Artikel_History_Superset #> function push-superset{ [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string] $supersetToCopy, [Parameter(mandatory=$false)] [string] $backupBehavior = "Auto" ) # Variables $tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica $clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup $skipBackup = $false $maxRetries = 5 $retryInterval = 10 # seconds $retryCount = 0 $success = $false log-out -msg "----------------------------------------------" log-out -msg "---- Starting push of $supersetToCopy to the cloud" log-out -msg "----------------------------------------------" ##do not alter below $serverInstance = "SWMDATASQLPRD01.centralinfra.net" $databaseName = $null $storageAccountName = "stbagspezlisteprdsql" $containerName = "sqlbakfiles" $resourceGroupName="rg-BAGSpezListePrd-SQL" $vaultName="kv-BAGSpezListePrd-bkp" $secretName="superuser" $blobPermissions="rwd" $expiryTime = (Get-Date).AddHours(1) # SAS token valid for 1 hour $backupFileName = "$supersetToCopy.bak" <# $mod = Get-Module -Name Az.Accounts if($null -eq $mod){ Install-Module -Name Az.KeyVault -Scope CurrentUser -force } $mod = Get-Module -Name Az.Storage if($null -eq $mod){ Install-Module -Name Az.Storage -Scope CurrentUser -Force } $mod = Get-Module -Name sqlserver if($null -eq $mod){ Install-Module -Name sqlserver -Scope CurrentUser -AllowClobber -force } #> #Managed instance related $MILogin='superuser' $MIPwd=$null #fetched from key vault later $MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net" #check for the origin of product_superset $query=" SELECT s.name AS superset, d.name AS srcDb FROM sys.databases s JOIN sys.databases d ON d.[database_id] = s.[source_database_id] WHERE s.name='$supersetToCopy' " $res = Invoke-Sqlcmd -ServerInstance $serverInstance -Query $query $databaseName = $res.srcDb if($null -eq $databaseName){ #superset is not a db snapshot... $databaseName = $supersetToCopy } # Log in to Azure $retryCount = 0 $success = $false while (-not $success -and $retryCount -lt $maxRetries) { try{ $null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d $success=$true } catch{ $retryCount++ Start-Sleep -Seconds $retryInterval } } if(-not $success){ write-error "Error authenticating to azure" exit 1 } # Get the storage account context $storageAccount = Get-AzStorageAccount -ResourceGroupName $resourceGroupName -Name $storageAccountName $context = $storageAccount.Context # Get the blob properties $backupFile = "https://$storageAccountName.blob.core.windows.net/$containerName/$backupFileName" if($backupAutoBehavior -eq "Auto") { try{ $blob = Get-AzStorageBlob -Container $containerName -Blob $backupFileName -Context $context } catch{ $blob = $null } if($null -eq $blob){ $skipBackup=$false log-out -msg "Blob does not exists, forcing backup" } else{ # Calculate the age of the blob $currentDate = Get-Date $blobLastModified = $blob | Select-Object -ExpandProperty LastModified $blobLastModifiedDateTime = [DateTime]::Parse($blobLastModified.ToString()) $blobAge = $currentDate - $blobLastModifiedDateTime if($blobAge.Days -eq 0){ $skipBackup=$true log-out -msg "Blob exists and is less than a day old, skipping backup" } else{ $skipBackup=$false log-out -msg "Blob exists and is older than a day, forcing backup" } } } else{ if($backupAutoBehavior -eq "Force"){ $skipBackup = $false log-out -msg "Forcing backup (param)" } if($backupAutoBehavior -eq "Skip"){ $skipBackup = $true log-out -msg "Skipping backup (param)" } else{ $skipBackup = $false log-out -msg "Forcing backup (empty param)" } } # Generate the SAS token $sasToken = New-AzStorageBlobSASToken -Context $context -Container $containerName -Blob $backupFileName -Permission $blobPermissions -ExpiryTime $expiryTime # Save token in db $sqlQuery = " IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName') BEGIN CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; END ELSE BEGIN ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; END " $retryCount = 0 $success = $false while (-not $success -and $retryCount -lt $maxRetries) { try{ Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery log-out -msg "Credential refreshed on $serverInstance" $success=$true } catch{ $retryCount++ Start-Sleep -Seconds $retryInterval } } if(-not $success){ write-error "Error refreshing credentials on $serverInstance" exit 1 } #Back Up Database $url=$backupFile $sqlQuery = " BACKUP DATABASE [$databaseName] TO URL = N'$url' WITH FORMAT, MEDIANAME = 'SQLServerBackups', NAME = 'Full Backup of $databaseName'; " if($false -eq $skipBackup){ $retryCount = 0 $success = $false while (-not $success -and $retryCount -lt $maxRetries) { try{ Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery log-out -msg "backup done" $success = $true } catch{ $retryCount ++ wait-Start-Sleep -Seconds $retryInterval } } if(-not $success){ write-error "Error backing up $databaseName" exit 1 } } else{ log-out -msg "Backup skipped." } #fetch the Managed Instance password $retryCount = 0 $success = $false while (-not $success -and $retryCount -lt $maxRetries) { try{ $MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText $success = $true } catch{ $retryCount ++ Start-Sleep -Seconds $retryInterval } } if(-not $success){ write-error "Error getting secret from key vault" exit 1 } #craft credential for MI connection $securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force $credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword #add sas token $sqlCred=" IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName') BEGIN CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; END ELSE BEGIN ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName] WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken'; END " $retryCount = 0 $success = $false while (-not $success -and $retryCount -lt $maxRetries) { try{ Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlCred -Credential $credentialMI log-out -msg "Credential refreshed on $MIInstance" $success=$true } catch{ $retryCount++ Start-Sleep -Seconds $retryInterval } } if(-not $success){ write-error "Error refreshing credentials on $MIInstance" exit 1 } #drop existing db $sqlDrop=" IF EXISTS( SELECT 1 FROM sys.databases d WHERE d.name ='$supersetToCopy' ) BEGIN DROP DATABASE $supersetToCopy; END " $retryCount = 0 $success = $false while (-not $success -and $retryCount -lt $maxRetries) { try{ Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlDrop -Credential $credentialMI log-out -msg "Dropped existing $supersetToCopy db (if needed)" $success=$true } catch{ $retryCount++ Start-Sleep -Seconds $retryInterval } } if(-not $success){ write-error "Error dropping existing db $supersetToCopy" exit 1 } #restore superset if not found on the MI instance $db_retry_count = 0 $db_retry_max_count = 5 $db_exists = find-db-exists-MI -superset $supersetToCopy while(($db_exists -eq $false) -and ($db_retry_count -lt $db_retry_max_count) ){ $url=$backupFile $sqlRestore="RESTORE DATABASE [$supersetToCopy] FROM URL = N'$url'; " $retryCount = 0 $success = $false while (-not $success -and $retryCount -lt $maxRetries) { try{ Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlRestore -Credential $credentialMI log-out -msg "Restored $supersetToCopy" $success=$true } catch{ $retryCount++ Start-Sleep -Seconds $retryInterval } } if(-not $success){ write-error "Error restoring db $supersetToCopy" exit 1 } $db_retry_count += 1 $db_exists = find-db-exists-MI -superset $supersetToCopy if(-not($db_exists)){ Start-Sleep -Seconds $retryInterval } } #check that after all tries, we have a db if(-not(find-db-exists-MI -superset $supersetToCopy)){ Write-Error "Superset $supersetToCopy is not present after restore !" exit 1 } #create user for sql-au_bag_apv $sqlUser=" IF NOT EXISTS ( SELECT * FROM sys.database_principals s WHERE s.name='sql-au_bag_apv' ) BEGIN CREATE USER [sql-au_bag_apv] FOR LOGIN [sql-au_bag_apv]; END " $retryCount = 0 $success = $false while (-not $success -and $retryCount -lt $maxRetries) { try{ Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlUser -Credential $credentialMI -Database $supersetToCopy log-out -msg "Created user for login [sql-au_bag_apv] in $supersetToCopy" $success=$true } catch{ $retryCount++ Start-Sleep -Seconds $retryInterval } } if(-not $success){ write-error "Error creating user in $supersetToCopy" exit 1 } #give db_datareader $sqlPerms=" IF '$supersetToCopy' = 'sl2007' BEGIN ALTER ROLE [db_owner] ADD MEMBER [sql-au_bag_apv]; END ELSE BEGIN ALTER ROLE [db_datareader] ADD MEMBER [sql-au_bag_apv]; ALTER ROLE [db_datawriter] ADD MEMBER [sql-au_bag_apv]; END GRANT EXECUTE TO [sql-au_bag_apv]; " $retryCount = 0 $success = $false while (-not $success -and $retryCount -lt $maxRetries) { try{ Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlPerms -Credential $credentialMI -Database $supersetToCopy log-out -msg "Added user [sql-au_bag_apv] with read and execute permissions to $supersetToCopy" $success=$true } catch{ $retryCount++ Start-Sleep -Seconds $retryInterval } } if(-not $success){ write-error "Error giving permissions to user in $supersetToCopy" exit 1 } } log-out -msg "Starting transfert of Artikel_History_Superset" push-superset -supersetToCopy Artikel_History_Superset -backupBehavior "Force" log-out -msg "Done" log-out -msg "Starting transfert of product_Superset" push-superset -supersetToCopy product_superset -backupBehavior "Force" log-out -msg "Done" log-out -msg "Starting transfert of artikel_superset" push-superset -supersetToCopy artikel_superset -backupBehavior "Force" log-out -msg "Done" log-out -msg "Starting transfert of artikel_superset" push-superset -supersetToCopy ODB_JobManager -backupBehavior "Force" log-out -msg "Done"