added powershell jobs deployed on prodDB to execute sync and production

HCI - BAG symmetricDS/proddb to mi backup.ps1

@@ -0,0 +1,453 @@
+#NOSQLPS
+
+function log-out{
+    [CmdletBinding()] 
+    param(   
+        [Parameter(Mandatory=$true)]
+        [string] $msg
+    )
+    Write-Output "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") :: $msg"
+}
+  
+<#
+.SYNOPSIS
+Push a superset from prodDB to the BAG Azure managed sql instance
+  
+.DESCRIPTION
+This function will do those operations:
+- Check if the superset given is a snapshot or a database
+if it's a snapshot, the source database behind the snapshot is backed up
+- Authenticate to Azur using a service principal (with a certificate authentication)
+- Create a SAS tocken from Azure Blob Storage 
+- Refresh credentials on prodDB to access the storage account
+- Check if a blob of the superset we want to push is more than 1 day old
+    - if no blob exists, force a backup
+    - if a blob exists but is aged of less than 1 full day, skip the backup
+    - if a blob exists but is older than 1 full day, force a new backup
+- Refresh the credential with the SAS token on the BAG managed sql instance
+- Drop the target database if it exists (we cannot restore over an existing db in managed instances)
+- Restore the backup in the cloud db
+- Create logins on the restored db for the login [sql-au_bag_apv]
+- Give db_datareader and EXECUTE permission on the restored db to [sql-au_bag_apv]
+  
+.PARAMETERS
+[string] $supersetToCopy
+The name of the superset to transfert. for exemple: product_superset
+  
+$backupBehavior
+auto = check age of blob and skip if backup is less than 1 day old, $true to force backup, $false to skip backup.
+  
+.EXAMPLES
+push-superset -supersetToCopy Artikel_History_Superset
+  
+#>
+function push-superset{
+    [CmdletBinding()] 
+    param(   
+        [Parameter(Mandatory=$true)]
+        [string] $supersetToCopy,
+
+        [Parameter(mandatory=$false)]
+        [string] $backupBehavior = "Auto"
+    )
+      
+    # Variables
+    $tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica
+    $clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup
+    $skipBackup = $false
+    
+    $maxRetries = 5
+    $retryInterval = 10 # seconds
+    $retryCount = 0
+    $success = $false
+
+  
+    log-out -msg "----------------------------------------------"
+    log-out -msg "---- Starting push of $supersetToCopy to the cloud"
+    log-out -msg "----------------------------------------------"
+
+    ##do not alter below
+    $serverInstance = "SWMDATASQLPRD01.centralinfra.net"
+    $databaseName = $null 
+    $storageAccountName = "stbagspezlisteprdsql"
+    $containerName = "sqlbakfiles"
+    $resourceGroupName="rg-BAGSpezListePrd-SQL"
+    $vaultName="kv-BAGSpezListePrd-bkp"
+    $secretName="superuser"
+    $blobPermissions="rwd"
+    $expiryTime = (Get-Date).AddHours(1)  # SAS token valid for 1 hour
+    $backupFileName = "$supersetToCopy.bak"
+
+	<#
+    $mod = Get-Module -Name Az.Accounts
+    if($null -eq $mod){
+        Install-Module -Name Az.KeyVault -Scope CurrentUser -force
+    }
+  
+    $mod = Get-Module -Name Az.Storage
+    if($null -eq $mod){
+          Install-Module -Name Az.Storage -Scope CurrentUser -Force
+    }
+  
+    $mod = Get-Module -Name sqlserver
+    if($null -eq $mod){
+        Install-Module -Name sqlserver -Scope CurrentUser -AllowClobber -force
+    }
+	#>
+  
+  
+    #Managed instance related
+    $MILogin='superuser'
+    $MIPwd=$null #fetched from key vault later
+    $MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net"
+
+    #check for the origin of product_superset
+    $query="
+    SELECT s.name AS superset, d.name AS srcDb
+    FROM sys.databases s
+        JOIN sys.databases d ON d.[database_id] = s.[source_database_id]
+    WHERE s.name='$supersetToCopy'
+    "
+    $res = Invoke-Sqlcmd -ServerInstance $serverInstance -Query $query
+    $databaseName = $res.srcDb
+    if($null -eq $databaseName){
+        #superset is not a db snapshot...
+        $databaseName = $supersetToCopy
+    }
+  
+    # Log in to Azure
+    $retryCount = 0
+    $success = $false
+    while (-not $success -and $retryCount -lt $maxRetries) {
+        try{
+            $null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d
+            $success=$true
+        }
+        catch{
+            $retryCount++
+            Start-Sleep -Seconds $retryInterval
+        }
+    }
+    
+    if(-not $success){
+        write-error "Error authenticating to azure"
+        exit 1
+    }
+    
+    # Get the storage account context
+    $storageAccount = Get-AzStorageAccount -ResourceGroupName $resourceGroupName -Name $storageAccountName
+    $context = $storageAccount.Context
+
+    # Get the blob properties
+    $backupFile = "https://$storageAccountName.blob.core.windows.net/$containerName/$backupFileName"
+    if($backupAutoBehavior -eq "Auto")
+    {
+        try{
+            $blob = Get-AzStorageBlob -Container $containerName -Blob $backupFileName -Context $context
+        }
+        catch{
+            $blob = $null
+        }
+
+        if($null -eq $blob){
+            $skipBackup=$false
+            log-out -msg "Blob does not exists, forcing backup"
+        }
+        else{
+            # Calculate the age of the blob
+            $currentDate = Get-Date
+            $blobLastModified = $blob | Select-Object -ExpandProperty  LastModified
+            $blobLastModifiedDateTime = [DateTime]::Parse($blobLastModified.ToString())
+            $blobAge = $currentDate - $blobLastModifiedDateTime
+
+            if($blobAge.Days -eq 0){
+                $skipBackup=$true
+                log-out -msg "Blob exists and is less than a day old, skipping backup"
+            }
+            else{
+                $skipBackup=$false
+                log-out -msg "Blob exists and is older than a day, forcing backup"
+            }
+        }
+    }
+    else{
+        if($backupAutoBehavior -eq "Force"){
+            $skipBackup = $false
+            log-out -msg "Forcing backup (param)"
+        }
+        
+        if($backupAutoBehavior -eq "Skip"){
+            $skipBackup = $true
+            log-out -msg "Skipping backup (param)"
+        }
+        else{
+            $skipBackup = $false
+            log-out -msg "Forcing backup (empty param)"
+
+        }
+    }
+  
+    # Generate the SAS token
+    $sasToken = New-AzStorageBlobSASToken -Context $context -Container $containerName -Blob $backupFileName -Permission $blobPermissions -ExpiryTime $expiryTime
+
+    # Save token in db 
+    $sqlQuery = "
+IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName')
+BEGIN
+    CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName]
+    WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken';
+END
+ELSE
+BEGIN
+    ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName]
+    WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken';
+END
+      "
+    $retryCount = 0
+    $success = $false
+
+    while (-not $success -and $retryCount -lt $maxRetries) {
+        try{
+            Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery
+            log-out -msg "Credential refreshed on $serverInstance"
+            $success=$true
+        }
+        catch{
+            $retryCount++
+            Start-Sleep -Seconds $retryInterval
+        }
+    }
+
+    if(-not $success){
+        write-error "Error refreshing credentials on $serverInstance"
+        exit 1
+    }
+  
+    #Back Up Database
+    $url=$backupFile
+    $sqlQuery = "
+BACKUP DATABASE [$databaseName]
+TO URL = N'$url'
+WITH FORMAT, MEDIANAME = 'SQLServerBackups', NAME = 'Full Backup of $databaseName';
+    "
+    if($false -eq $skipBackup){
+        $retryCount = 0
+        $success = $false
+        
+        while (-not $success -and $retryCount -lt $maxRetries) {
+            try{
+                Invoke-Sqlcmd -ServerInstance $serverInstance -Query $sqlQuery
+                log-out -msg "backup done"
+                $success = $true
+            }
+            catch{
+                $retryCount ++
+                wait-Start-Sleep -Seconds $retryInterval
+            }
+        }
+
+        if(-not $success){
+            write-error "Error backing up $databaseName"
+            exit 1
+        }
+    }
+    else{
+        log-out -msg "Backup skipped."
+    }
+
+    #fetch the Managed Instance password
+    $retryCount = 0
+    $success = $false
+    while (-not $success -and $retryCount -lt $maxRetries) {
+        try{
+            $MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText 
+            $success = $true
+        }
+        catch{
+            $retryCount ++
+            Start-Sleep -Seconds $retryInterval
+        }
+    }
+    if(-not $success){
+        write-error "Error getting secret from key vault"
+        exit 1
+    }
+
+    #craft credential for MI connection
+    $securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force
+    $credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword
+
+    #add sas token
+    $sqlCred="
+IF NOT EXISTS (SELECT * FROM sys.credentials WHERE name = 'https://$storageAccountName.blob.core.windows.net/$containerName')
+BEGIN
+    CREATE CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName]
+    WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken';
+END
+ELSE
+BEGIN
+    ALTER CREDENTIAL [https://$storageAccountName.blob.core.windows.net/$containerName]
+    WITH IDENTITY = 'SHARED ACCESS SIGNATURE', SECRET = '$sasToken';
+END
+    "
+    $retryCount = 0
+    $success = $false
+  
+    while (-not $success -and $retryCount -lt $maxRetries) {
+        try{
+            Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlCred -Credential $credentialMI
+            log-out -msg "Credential refreshed on $MIInstance"
+            $success=$true
+        }
+        catch{
+            $retryCount++
+            Start-Sleep -Seconds $retryInterval
+        }
+    }
+    
+    if(-not $success){
+        write-error "Error refreshing credentials on $MIInstance"
+        exit 1
+    }
+    
+  
+    #drop existing db
+    $sqlDrop="
+  
+IF EXISTS(
+    SELECT 1
+    FROM sys.databases d
+    WHERE d.name ='$supersetToCopy'
+)
+BEGIN
+    DROP DATABASE $supersetToCopy;
+END
+    "
+    $retryCount = 0
+    $success = $false
+    while (-not $success -and $retryCount -lt $maxRetries) {
+        try{
+            Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlDrop -Credential $credentialMI
+            log-out -msg "Dropped existing $supersetToCopy db (if needed)"        
+            $success=$true
+        }
+        catch{
+            $retryCount++
+            Start-Sleep -Seconds $retryInterval
+        }
+    }
+    
+    if(-not $success){
+        write-error "Error dropping existing db $supersetToCopy"
+        exit 1
+    }
+    
+
+    #restore superset
+    $url=$backupFile
+    $sqlRestore="RESTORE DATABASE [$supersetToCopy] FROM URL = N'$url'; "
+    $retryCount = 0
+    $success = $false
+    while (-not $success -and $retryCount -lt $maxRetries) {
+        try{
+            Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlRestore -Credential $credentialMI
+            log-out -msg "Restored $supersetToCopy"        
+            $success=$true
+        }
+        catch{
+            $retryCount++
+            Start-Sleep -Seconds $retryInterval
+        }
+    }
+
+    if(-not $success){
+        write-error "Error restoring db $supersetToCopy"
+        exit 1
+    }
+
+    
+
+    #create user for sql-au_bag_apv
+    $sqlUser="
+  
+IF NOT EXISTS (
+    SELECT * 
+    FROM sys.database_principals s
+    WHERE s.name='sql-au_bag_apv'
+)
+BEGIN
+    CREATE USER [sql-au_bag_apv] FOR LOGIN [sql-au_bag_apv];
+END
+    "
+    $retryCount = 0
+    $success = $false    
+    while (-not $success -and $retryCount -lt $maxRetries) {
+        try{
+            Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlUser -Credential $credentialMI -Database $supersetToCopy
+            log-out -msg "Created user for login [sql-au_bag_apv] in $supersetToCopy"
+            $success=$true
+        }
+        catch{
+            $retryCount++
+            Start-Sleep -Seconds $retryInterval
+        }
+    }
+
+    if(-not $success){
+        write-error "Error creating user in $supersetToCopy"
+        exit 1
+    }
+
+
+    #give db_datareader
+    $sqlPerms="
+  
+IF '$supersetToCopy' = 'sl2007'
+BEGIN
+    ALTER ROLE [db_owner] ADD MEMBER [sql-au_bag_apv];
+END
+ELSE
+BEGIN
+    ALTER ROLE [db_datareader] ADD MEMBER [sql-au_bag_apv];
+    ALTER ROLE [db_datawriter] ADD MEMBER [sql-au_bag_apv];
+END
+GRANT EXECUTE TO [sql-au_bag_apv];
+
+    "
+    $retryCount = 0
+    $success = $false
+    while (-not $success -and $retryCount -lt $maxRetries) {
+        try{
+            Invoke-Sqlcmd -ServerInstance $MIInstance -Query $sqlPerms -Credential $credentialMI -Database $supersetToCopy
+            log-out -msg "Added user [sql-au_bag_apv] with read and execute permissions to $supersetToCopy"
+            $success=$true
+        }
+        catch{
+            $retryCount++
+            Start-Sleep -Seconds $retryInterval
+        }
+    }
+
+    if(-not $success){
+        write-error "Error giving permissions to user in $supersetToCopy"
+        exit 1
+    }
+
+}
+
+log-out -msg "Starting transfert of Artikel_History_Superset"
+push-superset -supersetToCopy Artikel_History_Superset -backupBehavior "Force"
+log-out -msg "Done"
+
+log-out -msg "Starting transfert of product_Superset"
+push-superset -supersetToCopy product_superset -backupBehavior "Force"
+log-out -msg "Done"
+
+log-out -msg "Starting transfert of artikel_superset"
+push-superset -supersetToCopy artikel_superset -backupBehavior "Force"
+log-out -msg "Done"
+
+log-out -msg "Starting transfert of artikel_superset"
+push-superset -supersetToCopy ODB_JobManager -backupBehavior "Force"
+log-out -msg "Done"
+ 

HCI - BAG symmetricDS/sl2007 production.ps1

@@ -0,0 +1,118 @@
+# Variables
+$tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica
+$clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup
+$vaultName="kv-BAGSpezListePrd-bkp"
+$secretName="superuser"
+
+$maxRetries = 5
+$retryInterval = 10 # seconds
+$retryCount = 0
+$success = $false
+
+#Managed instance related
+$MILogin='superuser'
+$MIPwd=$null #fetched from key vault later
+$MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net"
+
+function log-out{
+    [CmdletBinding()] 
+    param(   
+        [Parameter(Mandatory=$true)]
+        [string] $msg
+    )
+    Write-Output "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") :: $msg"
+}
+
+# Log in to Azure
+$retryCount = 0
+$success = $false
+
+while (-not $success -and $retryCount -lt $maxRetries) {
+    try{
+        $null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d  
+        $success=$true  
+    }
+    catch{
+        $retryCount++
+        Start-Sleep -Seconds $retryInterval
+    }
+}
+
+if(-not $success){
+    write-error "Error auhthenticating to azure"
+    exit 1
+}
+  
+#fetch the Managed Instance password
+$retryCount = 0
+$success = $false
+
+while (-not $success -and $retryCount -lt $maxRetries) {
+    try{
+        $MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText 
+        $success=$true
+    }
+    catch{
+        $retryCount++
+        Start-Sleep -Seconds $retryInterval
+    }
+}
+
+if(-not $success){
+    write-error "Error fetching password from key vault"
+    exit 1
+}
+  
+#craft credential for MI connection
+$securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force
+$credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword
+
+#start sl2007.daily_batch_update
+log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") starting sl2007 daily update"
+$query="
+exec sl2007.dbo.usp_Daily_Batch_Update
+"
+$retryCount = 0
+$success = $false
+
+while (-not $success -and $retryCount -lt $maxRetries) {
+    try{
+        Invoke-Sqlcmd -ServerInstance $MIInstance -Query $query -Credential $credentialMI
+        log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") sl2007 daily update done"
+        $success=$true
+    }
+    catch{
+        $retryCount++
+        Start-Sleep -Seconds $retryInterval
+    }
+}
+
+if(-not $success){
+    write-error "Error starting usp_Daily_Batch_Update"
+    exit 1
+}
+
+#start sl2007_superset generation
+log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") starting sl2007_superset generation"
+$query="
+exec sl2007_superset.dbo.p_Fill__BAG__SL2007_Superset
+"
+$retryCount = 0
+$success = $false
+
+while (-not $success -and $retryCount -lt $maxRetries) {
+    try{
+        Invoke-Sqlcmd -ServerInstance $MIInstance -Query $query -Credential $credentialMI
+        log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") sl2007_superset generation done"
+        $success=$true
+    }
+    catch{
+        $retryCount++
+        Start-Sleep -Seconds $retryInterval
+    }
+}
+
+if(-not $success){
+    write-error "Error starting sl2007_suèerset production"
+    exit 1
+}

HCI - BAG symmetricDS/start snapshots sync.ps1

@@ -0,0 +1,115 @@
+# Variables
+$tenantId = "7844775a-a9cc-4c33-a5ae-36dcf6660f45" #Galenica
+$clientId = "d28076dd-2108-4718-802e-cd3c35fd5505" #pcpl-BAGSpezListePrd-DBBackup
+$vaultName="kv-BAGSpezListePrd-bkp"
+$secretName="superuser"
+
+$maxRetries = 5
+$retryInterval = 10 # seconds
+$retryCount = 0
+$success = $false
+
+#Managed instance related
+$MILogin='superuser'
+$MIPwd=$null #fetched from key vault later
+$MIInstance="sqlmi-bagspezlisteprd-sqlinstance.75ff9425ac13.database.windows.net"
+
+function log-out{
+    [CmdletBinding()] 
+    param(   
+        [Parameter(Mandatory=$true)]
+        [string] $msg
+    )
+    Write-Output "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") :: $msg"
+}
+
+while (-not $success -and $retryCount -lt $maxRetries) {
+    try{
+        $null = Connect-AzAccount -ServicePrincipal -ApplicationId $clientId -Tenant $tenantId -Subscription BAGSpezListePrd -CertificateThumbprint 7bd45f67999015c7742db25efc86bae97590c57d    
+        $success = $true
+    }
+    catch{
+        $retryCount++
+        Start-Sleep -Seconds $retryInterval
+    }
+}
+
+if(-not $success){
+    write-error "Error auhthenticating to azure"
+    exit 1
+}
+  
+#fetch the Managed Instance password
+$retryCount = 0
+$success = $false
+
+while (-not $success -and $retryCount -lt $maxRetries) {
+    try{
+        $MIPwd = Get-AzKeyVaultSecret -VaultName $vaultName -Name $secretName -AsPlainText 
+        $success = $true
+    }
+    catch{
+        $retryCount++
+        Start-Sleep -Seconds $retryInterval
+    }
+}
+
+if(-not $success){
+    write-error "Error fetching password from key vault"
+    exit 1
+}
+  
+#craft credential for MI connection
+$securePassword = ConvertTo-SecureString $MIPwd -AsPlainText -Force
+$credentialMI = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $MILogin, $securePassword
+
+#start sl2007.daily_batch_update
+log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") starting sl2007 snapshot"
+$query="
+exec dba.dbo.start_snapshot @published_db='sl2007', @debug=0
+"
+$retryCount = 0
+$success = $false
+
+while (-not $success -and $retryCount -lt $maxRetries) {
+    try{
+        Invoke-Sqlcmd -ServerInstance $MIInstance -Query $query -Credential $credentialMI
+        log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") sl2007 snapshot done"
+        $success = $true
+    }
+    catch{
+        $retryCount++
+        Start-Sleep -Seconds $retryInterval
+    }
+}
+
+if(-not $success){
+    write-error "Error generating sl2007 snapshot"
+    exit 1
+}
+
+
+#start sl2007_superset generation
+log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") starting sl2007_superset snapshot"
+$query="
+exec dba.dbo.start_snapshot @published_db='sl2007_superset', @debug=0
+"
+$retryCount = 0
+$success = $false
+
+while (-not $success -and $retryCount -lt $maxRetries) {
+    try{
+        Invoke-Sqlcmd -ServerInstance $MIInstance -Query $query -Credential $credentialMI
+        log-out -msg "$(Get-Date -Format "yyyy-MM-dd HH:mm:ss") sl2007_superset snapshot done"
+        $success=$true
+    }
+    catch{
+        $retryCount++
+        Start-Sleep -Seconds $retryInterval
+    }
+}
+
+if(-not $success){
+    write-error "Error generating sl2007_superset snapshot"
+    exit 1
+}