453 lines
14 KiB
PowerShell
453 lines
14 KiB
PowerShell
#NOSQLPS
|
|
|
|
function log-out{
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory=$true)]
|
|
[string] $msg
|
|
)
|
|
Write-Output "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") :: $msg"
|
|
}
|
|
|
|
<#
|
|
.SYNOPSIS
|
|
Push a superset from prodDB to the BAG Azure managed sql instance
|
|
|
|
.DESCRIPTION
|
|
This function will do those operations:
|
|
- Check if the superset given is a snapshot or a database
|
|
if it's a snapshot, the source database behind the snapshot is backed up
|
|
- Authenticate to Azur using a service principal (with a certificate authentication)
|
|
- Create a SAS tocken from Azure Blob Storage
|
|
- Refresh credentials on prodDB to access the storage account
|
|
- Check if a blob of the superset we want to push is more than 1 day old
|
|
- if no blob exists, force a backup
|
|
- if a blob exists but is aged of less than 1 full day, skip the backup
|
|
- if a blob exists but is older than 1 full day, force a new backup
|
|
- Refresh the credential with the SAS token on the BAG managed sql instance
|
|
- Drop the target database if it exists (we cannot restore over an existing db in managed instances)
|
|
- Restore the backup in the cloud db
|
|
- Create logins on the restored db for the login [sql-au_bag_apv]
|
|
- Give db_datareader and EXECUTE permission on the restored db to [sql-au_bag_apv]
|
|
|
|
.PARAMETERS
|
|
[string] $supersetToCopy
|
|
The name of the superset to transfert. for exemple: product_superset
|
|
|
|
$backupBehavior
|
|
auto = check age of blob and skip if backup is less than 1 day old, $true to force backup, $false to skip backup.
|
|
|
|
.EXAMPLES
|
|
push-superset -supersetToCopy Artikel_History_Superset
|
|
|
|
#>
|
|
function push-superset{
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory=$true)]
|
|
[string] $supersetToCopy,
|
|
|
|
[Parameter(mandatory=$false)]
|
|
[string] $backupBehavior = "Auto"
|
|
)
|
|
|
|
# Variables
|
|
$tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica
|
|
$clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup
|
|
$skipBackup = $false
|
|
|
|
$maxRetries = 5
|
|
$retryInterval = 10 # seconds
|
|
$retryCount = 0
|
|
$success = $false
|
|
|
|
|
|
log-out -msg "----------------------------------------------"
|
|
log-out -msg "---- Starting push of $supersetToCopy to the cloud"
|
|
log-out -msg "----------------------------------------------"
|
|
|
|
##do not alter below
|
|
$serverInstance = "SWMDATASQLPRD01.centralinfra.net"
|
|
$databaseName = $null
|
|
$storageAccountName = "stbagspezlisteprdsql"
|
|
$containerName = "sqlbakfiles"
|
|
$resourceGroupName="rg-BAGSpezListePrd-SQL"
|
|
$vaultName="kv-BAGSpezListePrd-bkp"
|
|
$secretName="superuser"
|
|
$blobPermissions="rwd"
|
|
$expiryTime = (Get-Date).AddHours(1) # SAS token valid for 1 hour
|
|
$backupFileName = "$supersetToCopy.bak"
|
|
|
|
<#
|
|
$mod = Get-Module -Name Az.Accounts
|
|
if($null -eq $mod){
|
|
Install-Module -Name Az.KeyVault -Scope CurrentUser -force
|
|
}
|
|
|
|
$mod = Get-Module -Name Az.Storage
|
|
if($null -eq $mod){
|
|
Install-Module -Name Az.Storage -Scope CurrentUser -Force
|
|
}
|
|
|
|
$mod = Get-Module -Name sqlserver
|
|
if($null -eq $mod){
|
|
Install-Module -Name sqlserver -Scope CurrentUser -AllowClobber -force
|
|
}
|
|
#>
|
|
|
|
|
|
#Managed instance related
|
|
$MILogin='superuser'
|
|
$MIPwd=$null #fetched from key vault later
|
|
$MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net"
|
|
|
|
#check for the origin of product_superset
|
|
$query="
|
|
SELECT s.name AS superset, d.name AS srcDb
|
|
FROM sys.databases s
|
|
JOIN sys.databases d ON d.[database_id] = s.[source_database_id]
|
|
WHERE s.name='$supersetToCopy'
|
|
"
|
|
$res = Invoke-Sqlcmd -ServerInstance $serverInstance -Query $query
|
|
$databaseName = $res.srcDb
|
|
if($null -eq $databaseName){
|
|
#superset is not a db snapshot...
|
|
$databaseName = $supersetToCopy
|
|
}
|
|
|
|
# Log in to Azure
|
|
$retryCount = 0
|
|
$success = $false
|
|
while (-not $success -and $retryCount -lt $maxRetries) {
|
|
try{
|
|
$null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d
|
|
$success=$true
|
|
}
|
|
catch{
|
|
$retryCount++
|
|
Start-Sleep -Seconds $retryInterval
|
|
}
|
|
}
|
|
|
|
if(-not $success){
|
|
write-error "Error authenticating to azure"
|
|
exit 1
|
|
}
|
|
|
|
# Get the storage account context
|
|
$storageAccount = Get-AzStorageAccount -ResourceGroupName $resourceGroupName -Name $storageAccountName
|
|
$context = $storageAccount.Context
|
|
|
|
# Get the blob properties
|
|
$backupFile = "https://$storageAccountName.blob.core.windows.net/$containerName/$backupFileName"
|
|
if($backupAutoBehavior -eq "Auto")
|
|
{
|
|
try{
|
|
$blob = Get-AzStorageBlob -Container $containerName -Blob $backupFileName -Context $context
|
|
}
|
|
catch{
|
|
$blob = $null
|
|
}
|
|
|
|
if($null -eq $blob){
|
|
$skipBackup=$false
|
|
log-out -msg "Blob does not exists, forcing backup"
|
|
}
|
|
else{
|
|
# Calculate the age of the blob
|
|
$currentDate = Get-Date
|
|
$blobLastModified = $blob | Select-Object -ExpandProperty LastModified
|
|
$blobLastModifiedDateTime = [DateTime]::Parse($blobLastModified.ToString())
|
|
$blobAge = $currentDate - $blobLastModifiedDateTime
|
|
|
|
if($blobAge.Days -eq 0){
|
|
$skipBackup=$true
|
|
log-out -msg "Blob exists and is less than a day old, skipping backup"
|
|
}
|
|
else{
|
|
$skipBackup=$false
|
|
log-out -msg "Blob exists and is older than a day, forcing backup"
|
|
}
|
|
}
|
|
}
|
|
else{
|
|
if($backupAutoBehavior -eq "Force"){
|
|
$skipBackup = $false
|
|
log-out -msg "Forcing backup (param)"
|
|
}
|
|
|
|
if($backupAutoBehavior -eq "Skip"){
|
|
$skipBackup = $true
|
|
log-out -msg "Skipping backup (param)"
|
|
}
|
|
else{
|
|
$skipBackup = $false
|
|
log-out -msg "Forcing backup (empty param)"
|
|
|
|
}
|
|
}
|
|
|
|
# Generate the SAS token
|
|
$sasToken = New-AzStorageBlobSASToken -Context $context -Container $containerName -Blob $backupFileName -Permission $blobPermissions -ExpiryTime $expiryTime
|
|
|
|
# Save token in db
|
|
$sqlQuery = "
|
|
IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName')
|
|
BEGIN
|
|
CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName]
|
|
WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken';
|
|
END
|
|
ELSE
|
|
BEGIN
|
|
ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName]
|
|
WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken';
|
|
END
|
|
"
|
|
$retryCount = 0
|
|
$success = $false
|
|
|
|
while (-not $success -and $retryCount -lt $maxRetries) {
|
|
try{
|
|
Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery
|
|
log-out -msg "Credential refreshed on $serverInstance"
|
|
$success=$true
|
|
}
|
|
catch{
|
|
$retryCount++
|
|
Start-Sleep -Seconds $retryInterval
|
|
}
|
|
}
|
|
|
|
if(-not $success){
|
|
write-error "Error refreshing credentials on $serverInstance"
|
|
exit 1
|
|
}
|
|
|
|
#Back Up Database
|
|
$url=$backupFile
|
|
$sqlQuery = "
|
|
BACKUP DATABASE [$databaseName]
|
|
TO URL = N'$url'
|
|
WITH FORMAT, MEDIANAME = 'SQLServerBackups', NAME = 'Full Backup of $databaseName';
|
|
"
|
|
if($false -eq $skipBackup){
|
|
$retryCount = 0
|
|
$success = $false
|
|
|
|
while (-not $success -and $retryCount -lt $maxRetries) {
|
|
try{
|
|
Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery
|
|
log-out -msg "backup done"
|
|
$success = $true
|
|
}
|
|
catch{
|
|
$retryCount ++
|
|
wait-Start-Sleep -Seconds $retryInterval
|
|
}
|
|
}
|
|
|
|
if(-not $success){
|
|
write-error "Error backing up $databaseName"
|
|
exit 1
|
|
}
|
|
}
|
|
else{
|
|
log-out -msg "Backup skipped."
|
|
}
|
|
|
|
#fetch the Managed Instance password
|
|
$retryCount = 0
|
|
$success = $false
|
|
while (-not $success -and $retryCount -lt $maxRetries) {
|
|
try{
|
|
$MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText
|
|
$success = $true
|
|
}
|
|
catch{
|
|
$retryCount ++
|
|
Start-Sleep -Seconds $retryInterval
|
|
}
|
|
}
|
|
if(-not $success){
|
|
write-error "Error getting secret from key vault"
|
|
exit 1
|
|
}
|
|
|
|
#craft credential for MI connection
|
|
$securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force
|
|
$credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword
|
|
|
|
#add sas token
|
|
$sqlCred="
|
|
IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName')
|
|
BEGIN
|
|
CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName]
|
|
WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken';
|
|
END
|
|
ELSE
|
|
BEGIN
|
|
ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName]
|
|
WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken';
|
|
END
|
|
"
|
|
$retryCount = 0
|
|
$success = $false
|
|
|
|
while (-not $success -and $retryCount -lt $maxRetries) {
|
|
try{
|
|
Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlCred -Credential $credentialMI
|
|
log-out -msg "Credential refreshed on $MIInstance"
|
|
$success=$true
|
|
}
|
|
catch{
|
|
$retryCount++
|
|
Start-Sleep -Seconds $retryInterval
|
|
}
|
|
}
|
|
|
|
if(-not $success){
|
|
write-error "Error refreshing credentials on $MIInstance"
|
|
exit 1
|
|
}
|
|
|
|
|
|
#drop existing db
|
|
$sqlDrop="
|
|
|
|
IF EXISTS(
|
|
SELECT 1
|
|
FROM sys.databases d
|
|
WHERE d.name ='$supersetToCopy'
|
|
)
|
|
BEGIN
|
|
DROP DATABASE $supersetToCopy;
|
|
END
|
|
"
|
|
$retryCount = 0
|
|
$success = $false
|
|
while (-not $success -and $retryCount -lt $maxRetries) {
|
|
try{
|
|
Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlDrop -Credential $credentialMI
|
|
log-out -msg "Dropped existing $supersetToCopy db (if needed)"
|
|
$success=$true
|
|
}
|
|
catch{
|
|
$retryCount++
|
|
Start-Sleep -Seconds $retryInterval
|
|
}
|
|
}
|
|
|
|
if(-not $success){
|
|
write-error "Error dropping existing db $supersetToCopy"
|
|
exit 1
|
|
}
|
|
|
|
|
|
#restore superset
|
|
$url=$backupFile
|
|
$sqlRestore="RESTORE DATABASE [$supersetToCopy] FROM URL = N'$url'; "
|
|
$retryCount = 0
|
|
$success = $false
|
|
while (-not $success -and $retryCount -lt $maxRetries) {
|
|
try{
|
|
Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlRestore -Credential $credentialMI
|
|
log-out -msg "Restored $supersetToCopy"
|
|
$success=$true
|
|
}
|
|
catch{
|
|
$retryCount++
|
|
Start-Sleep -Seconds $retryInterval
|
|
}
|
|
}
|
|
|
|
if(-not $success){
|
|
write-error "Error restoring db $supersetToCopy"
|
|
exit 1
|
|
}
|
|
|
|
|
|
|
|
#create user for sql-au_bag_apv
|
|
$sqlUser="
|
|
|
|
IF NOT EXISTS (
|
|
SELECT *
|
|
FROM sys.database_principals s
|
|
WHERE s.name='sql-au_bag_apv'
|
|
)
|
|
BEGIN
|
|
CREATE USER [sql-au_bag_apv] FOR LOGIN [sql-au_bag_apv];
|
|
END
|
|
"
|
|
$retryCount = 0
|
|
$success = $false
|
|
while (-not $success -and $retryCount -lt $maxRetries) {
|
|
try{
|
|
Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlUser -Credential $credentialMI -Database $supersetToCopy
|
|
log-out -msg "Created user for login [sql-au_bag_apv] in $supersetToCopy"
|
|
$success=$true
|
|
}
|
|
catch{
|
|
$retryCount++
|
|
Start-Sleep -Seconds $retryInterval
|
|
}
|
|
}
|
|
|
|
if(-not $success){
|
|
write-error "Error creating user in $supersetToCopy"
|
|
exit 1
|
|
}
|
|
|
|
|
|
#give db_datareader
|
|
$sqlPerms="
|
|
|
|
IF '$supersetToCopy' = 'sl2007'
|
|
BEGIN
|
|
ALTER ROLE [db_owner] ADD MEMBER [sql-au_bag_apv];
|
|
END
|
|
ELSE
|
|
BEGIN
|
|
ALTER ROLE [db_datareader] ADD MEMBER [sql-au_bag_apv];
|
|
ALTER ROLE [db_datawriter] ADD MEMBER [sql-au_bag_apv];
|
|
END
|
|
GRANT EXECUTE TO [sql-au_bag_apv];
|
|
|
|
"
|
|
$retryCount = 0
|
|
$success = $false
|
|
while (-not $success -and $retryCount -lt $maxRetries) {
|
|
try{
|
|
Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlPerms -Credential $credentialMI -Database $supersetToCopy
|
|
log-out -msg "Added user [sql-au_bag_apv] with read and execute permissions to $supersetToCopy"
|
|
$success=$true
|
|
}
|
|
catch{
|
|
$retryCount++
|
|
Start-Sleep -Seconds $retryInterval
|
|
}
|
|
}
|
|
|
|
if(-not $success){
|
|
write-error "Error giving permissions to user in $supersetToCopy"
|
|
exit 1
|
|
}
|
|
|
|
}
|
|
|
|
log-out -msg "Starting transfert of Artikel_History_Superset"
|
|
push-superset -supersetToCopy Artikel_History_Superset -backupBehavior "Force"
|
|
log-out -msg "Done"
|
|
|
|
log-out -msg "Starting transfert of product_Superset"
|
|
push-superset -supersetToCopy product_superset -backupBehavior "Force"
|
|
log-out -msg "Done"
|
|
|
|
log-out -msg "Starting transfert of artikel_superset"
|
|
push-superset -supersetToCopy artikel_superset -backupBehavior "Force"
|
|
log-out -msg "Done"
|
|
|
|
log-out -msg "Starting transfert of artikel_superset"
|
|
push-superset -supersetToCopy ODB_JobManager -backupBehavior "Force"
|
|
log-out -msg "Done"
|
|
|