thierry/sql-scripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync classification scripts

Browse Source
This commit is contained in:
Thierry Schork
2023-03-31 08:00:35 +02:00
parent ed02942184
commit 16640f9512
3 changed files with 603 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
CLASS - auto classify data.sql
View File

@@ -2,6 +2,31 @@
Propose data classification to all columns in database Propose data classification to all columns in database
This script is to help classify data in the database.
The logic is copied from the Sql Server Management Studio logic and extended.
This script is compatible with the SSMS gui and reports.
!!!
A couple variables are present around line 230.
They cannot be put higher in the script, please review them before launching the script.
!!!
The logic is:
* We collect a list of pattern to match against fields in the database. Those patterns can add or retract matches.
* If the flag @searchInFields is true, a check against the content of every fields containing text is also run
Only fields with a max lenght > 90 characters are checked.
* Only tables are checked, views are ignored
* Tables with no rows are ignored and removed from the matches
* every fields with a name finishing with "id" are removed from the results
* every primary key and foreign keys column are excluded from the results
* only columns with alpha numeric or numeric content are included
* computed columns are skipped
* a list of unused tables in the arizona database have been identified in the US OCTP-724
Those tables are excluded from the results as well.
It will set classification as:
category: "Galenica Used Personal Informations" with the sensitivity: "Confidential" on every matched rows
category: "Other" with sensitivity: "General" on every other columns
Creation : 29.03.2023 / TSC Creation : 29.03.2023 / TSC
Modifications: Modifications:
@@ -177,6 +202,8 @@ DECLARE @column_name VARCHAR(100);
DECLARE @data_type VARCHAR(100); DECLARE @data_type VARCHAR(100);
DECLARE @searchInFields BIT = 1; DECLARE @searchInFields BIT = 1;
DECLARE @assignUnmatchedColumns BIT = 1; DECLARE @assignUnmatchedColumns BIT = 1;
DECLARE @matchedType VARCHAR(255);
DECLARE @unmatchedType VARCHAR(255);
DECLARE @useSql2019Syntax BIT = CASE DECLARE @useSql2019Syntax BIT = CASE
WHEN CAST(SERVERPROPERTY('productversion') AS VARCHAR(2)) >= 15 THEN 1 WHEN CAST(SERVERPROPERTY('productversion') AS VARCHAR(2)) >= 15 THEN 1
ELSE 0 ELSE 0
@@ -214,7 +241,7 @@ If true, search inside text fields longer than 90 characters for occurence of th
Only non computed columns of tables are checked (views are ignored, computed columns are ignored) Only non computed columns of tables are checked (views are ignored, computed columns are ignored)
If false, the search inside the fields is skipped. If false, the search inside the fields is skipped.
*/ */
SET @searchInFields = 0; SET @searchInFields = 01;
/* /*
if true, every columns that is not matched by a pattern will be marked as category: Other, sensitivity: General if true, every columns that is not matched by a pattern will be marked as category: Other, sensitivity: General
@@ -222,6 +249,10 @@ if false, no classification will be set on the column. Existing classification w
*/ */
SET @assignUnmatchedColumns = 01; SET @assignUnmatchedColumns = 01;
/* The type applied to recognized fields */
SET @matchedType = 'Galenica Used Personal Informations';
SET @unmatchedType = 'Other';
--#region populate patterns --#region populate patterns
INSERT INTO @Tblpattern ([pattern]) INSERT INTO @Tblpattern ([pattern])
VALUES('%last%name%') VALUES('%last%name%')
@@ -379,7 +410,9 @@ BEGIN
AND t.[name] = c.[TABLE_NAME] AND t.[name] = c.[TABLE_NAME]
AND c1.[name] = c.[COLUMN_NAME] AND c1.[name] = c.[COLUMN_NAME]
) )
/* do not process known tables not used anymore. see OCTP-724 */ /* do not process known tables not used anymore in the arizona database. see OCTP-724 */
AND (
DB_NAME()='arizona'
AND c.[TABLE_SCHEMA]+'.'+c.[TABLE_NAME] NOT IN( AND c.[TABLE_SCHEMA]+'.'+c.[TABLE_NAME] NOT IN(
'dbo.TT_File_Transfer', 'dbo.TT_File_Transfer',
'dbo.pfs_gdi$', 'dbo.pfs_gdi$',
@@ -810,6 +843,7 @@ BEGIN
'dbo.Bmc_application_default_backup_vesta_migration_lot3', 'dbo.Bmc_application_default_backup_vesta_migration_lot3',
'dbo.TT_Bmc_application_default_backup_vesta_migration_lot3' 'dbo.TT_Bmc_application_default_backup_vesta_migration_lot3'
) )
)
; ;
@@ -940,7 +974,7 @@ PRINT CONVERT(VARCHAR(20), CURRENT_TIMESTAMP, 114)+' - Assign classification on
DECLARE crsr_classify_active CURSOR FAST_FORWARD READ_ONLY FOR DECLARE crsr_classify_active CURSOR FAST_FORWARD READ_ONLY FOR
--SELECT 'ADD SENSITIVITY CLASSIFICATION TO ['+c.[table_schema]+'].['+c.[table_name]+'].['+c.[column_name]+'] WITH ( LABEL=''Confidential'', INFORMATION_TYPE=''Galenica Used Personal Informations'', RANK=High );' AS command --SELECT 'ADD SENSITIVITY CLASSIFICATION TO ['+c.[table_schema]+'].['+c.[table_name]+'].['+c.[column_name]+'] WITH ( LABEL=''Confidential'', INFORMATION_TYPE=''Galenica Used Personal Informations'', RANK=High );' AS command
SELECT 'EXEC dbo.setSensitivity @schemaName = '''+ c.[TABLE_SCHEMA] +''', @tableName = '''+[c].[TABLE_NAME]+''', @columnName = '''+[c].[COLUMN_NAME]+''', @category = ''Galenica Used Personal Informations'', @sensitivity = ''Confidential'', @rank = ''High'', @removeClassification = 0 ' SELECT 'EXEC dbo.setSensitivity @schemaName = '''+ c.[TABLE_SCHEMA] +''', @tableName = '''+[c].[TABLE_NAME]+''', @columnName = '''+[c].[COLUMN_NAME]+''', @category = '''+@matchedType+''', @sensitivity = ''Confidential'', @rank = ''High'', @removeClassification = 0 '
FROM [#classification] c; FROM [#classification] c;
OPEN crsr_classify_active OPEN crsr_classify_active
@@ -1028,7 +1062,7 @@ EXEC(@cmd);
DECLARE crsr_classify_other CURSOR FAST_FORWARD READ_ONLY FOR DECLARE crsr_classify_other CURSOR FAST_FORWARD READ_ONLY FOR
--SELECT 'ADD SENSITIVITY CLASSIFICATION TO ['+[ic].[TABLE_SCHEMA]+'].['+[ic].[TABLE_NAME]+'].['+[ic].[COLUMN_NAME]+'] WITH ( LABEL=''General'', INFORMATION_TYPE=''Other'', RANK=Low )' --SELECT 'ADD SENSITIVITY CLASSIFICATION TO ['+[ic].[TABLE_SCHEMA]+'].['+[ic].[TABLE_NAME]+'].['+[ic].[COLUMN_NAME]+'] WITH ( LABEL=''General'', INFORMATION_TYPE=''Other'', RANK=Low )'
SELECT 'EXEC dbo.setSensitivity @schemaName = '''+ic.[TABLE_SCHEMA]+''', @tableName = '''+[ic].[TABLE_NAME]+''', @columnName = '''+[ic].[COLUMN_NAME]+''', @category = ''Other'', @sensitivity = ''General'', @rank = ''Low'', @removeClassification='+CAST(@assignUnmatchedColumns ^ 1 AS NVARCHAR(2))+' ' SELECT 'EXEC dbo.setSensitivity @schemaName = '''+ic.[TABLE_SCHEMA]+''', @tableName = '''+[ic].[TABLE_NAME]+''', @columnName = '''+[ic].[COLUMN_NAME]+''', @category = ''Other'', @sensitivity = '''+@unmatchedType+''', @rank = ''Low'', @removeClassification='+CAST(@assignUnmatchedColumns ^ 1 AS NVARCHAR(2))+' '
FROM [INFORMATION_SCHEMA].[COLUMNS] ic FROM [INFORMATION_SCHEMA].[COLUMNS] ic
JOIN [INFORMATION_SCHEMA].[TABLES] it ON it.[TABLE_SCHEMA] = ic.[TABLE_SCHEMA] AND it.[TABLE_NAME] = ic.[TABLE_NAME] JOIN [INFORMATION_SCHEMA].[TABLES] it ON it.[TABLE_SCHEMA] = ic.[TABLE_SCHEMA] AND it.[TABLE_NAME] = ic.[TABLE_NAME]
WHERE [it].[TABLE_TYPE] = 'BASE TABLE' WHERE [it].[TABLE_TYPE] = 'BASE TABLE'
@@ -1044,7 +1078,7 @@ DECLARE crsr_classify_other CURSOR FAST_FORWARD READ_ONLY FOR
WHERE ec.[table_schema] = ic.[TABLE_SCHEMA] WHERE ec.[table_schema] = ic.[TABLE_SCHEMA]
AND ec.[table_name] = ic.[TABLE_NAME] AND ec.[table_name] = ic.[TABLE_NAME]
AND ec.[column_name] = ic.[COLUMN_NAME] AND ec.[column_name] = ic.[COLUMN_NAME]
AND ec.[category] = 'Galenica Used Personal Informations' AND ec.[category] = @matchedType
) )
AND NOT EXISTS( AND NOT EXISTS(
/* ignore computed columns */ /* ignore computed columns */
@@ -1071,21 +1105,67 @@ CLOSE crsr_classify_other
DEALLOCATE crsr_classify_other DEALLOCATE crsr_classify_other
--#endregion --#endregion
---- sql 2019 or later only /* check classification recorded */
-- SELECT SCHEMA_NAME([O].[schema_id]) AS schema_name IF @useSql2019Syntax = 1
-- ,CAST(O.name AS VARCHAR(255)) AS table_name BEGIN
-- ,CAST(C.name AS VARCHAR(255)) AS column_name EXEC('
-- ,CAST(sc.information_type AS VARCHAR(255)) as information_type SELECT SCHEMA_NAME([O].[schema_id]) AS schema_name
-- ,CAST(sc.label AS VARCHAR(255)) as label ,CAST(O.name AS VARCHAR(255)) AS table_name
-- ,CAST(sc.rank_desc AS VARCHAR(255)) as rank_desc ,CAST(C.name AS VARCHAR(255)) AS column_name
--FROM sys.sensitivity_classifications sc ,CAST(sc.information_type AS VARCHAR(255)) as information_type
-- JOIN sys.objects O ,CAST(sc.label AS VARCHAR(255)) as label
-- ON [sc].[major_id] = O.object_id ,CAST(sc.rank_desc AS VARCHAR(255)) as rank_desc
-- JOIN sys.columns C FROM sys.sensitivity_classifications sc
-- ON [sc].[major_id] = C.object_id JOIN sys.objects O
-- AND [sc].[minor_id] = [C].[column_id] ON [sc].[major_id] = O.object_id
JOIN sys.columns C
ON [sc].[major_id] = C.object_id
AND [sc].[minor_id] = [C].[column_id]
');
END
ELSE
BEGIN
SELECT
schema_name(O.schema_id) AS schema_name,
O.[name] AS table_name,
C.[name] AS column_name,
[EP].[information_type],
[EP].[sensitivity_label]
FROM
(
SELECT
IT.major_id,
IT.minor_id,
IT.information_type,
L.sensitivity_label
FROM
(
SELECT
major_id,
minor_id,
value AS information_type
FROM sys.extended_properties
WHERE NAME = 'sys_information_type_name'
) IT
FULL OUTER JOIN
(
SELECT
major_id,
minor_id,
value AS sensitivity_label
FROM sys.extended_properties
WHERE NAME = 'sys_sensitivity_label_name'
) L
ON IT.major_id = L.major_id AND IT.minor_id = L.minor_id
) EP
JOIN sys.objects O
ON EP.major_id = O.object_id
JOIN sys.columns C
ON EP.major_id = C.object_id AND EP.minor_id = C.column_id
;
END
/* see what has been recognized as sensitive */
SELECT * SELECT *
FROM [#classification] c FROM [#classification] c
ORDER BY [c].[table_schema], [c].[table_name], [c].[column_name] ORDER BY [c].[table_schema], [c].[table_name], [c].[column_name]

23
CLASS - chaeck and export classification.sql
View File

@@ -1,23 +0,0 @@
SELECT SCHEMA_NAME([O].[schema_id]) AS schema_name,
O.name AS table_name,
C.name AS column_name,
[sc].[information_type],
[sc].[label],
[sc].[rank],
[sc].[rank_desc]
,'ADD SENSITIVITY CLASSIFICATION TO ['+CAST(SCHEMA_NAME([O].[schema_id]) AS NVARCHAR(100))+'].['+CAST(o.[name] AS NVARCHAR(100))+'].['+CAST(c.name AS NVARCHAR(100))+'] WITH ( LABEL='''+CAST(sc.[label] AS NVARCHAR(100))+''', INFORMATION_TYPE='''+CAST(sc.[information_type] AS NVARCHAR(100))+''' '
+ CASE
WHEN sc.[rank_desc] IS NOT NULL THEN ', RANK='+sc.[rank_desc]
ELSE ''
END
+')'
--sc.*
FROM sys.sensitivity_classifications sc
JOIN sys.objects O
ON [sc].[major_id] = O.object_id
JOIN sys.columns C
ON [sc].[major_id] = C.object_id
AND [sc].[minor_id] = [C].[column_id]
--WHERE CAST(sc.[information_type] AS VARCHAR(500)) <> 'Other'
--WHERE [O].[name] LIKE '[IIICommon_Bank_PTT_Master]'
ORDER BY CAST([sc].[information_type] AS VARCHAR(500)), [schema_name], [table_name], [column_name];

74
CLASS - check and export classification.sql Normal file
View File

@@ -0,0 +1,74 @@
DECLARE @useSql2019Syntax BIT = CASE
WHEN CAST(SERVERPROPERTY('productversion') AS VARCHAR(2)) >= 15 THEN 1
ELSE 0
END ;
IF @useSql2019Syntax = 1
BEGIN
EXEC ('
SELECT SCHEMA_NAME([O].[schema_id]) AS schema_name,
O.name AS table_name,
C.name AS column_name,
[sc].[information_type],
[sc].[label],
[sc].[rank],
[sc].[rank_desc]
,''ADD SENSITIVITY CLASSIFICATION TO [''+CAST(SCHEMA_NAME([O].[schema_id]) AS NVARCHAR(100))+''].[''+CAST(o.[name] AS NVARCHAR(100))+''].[''+CAST(c.name AS NVARCHAR(100))+''] WITH ( LABEL=''''''+CAST(sc.[label] AS NVARCHAR(100))+'''''', INFORMATION_TYPE=''''''+CAST(sc.[information_type] AS NVARCHAR(100))+'''''' ''
+ CASE
WHEN sc.[rank_desc] IS NOT NULL THEN '', RANK=''+sc.[rank_desc]
ELSE ''''
END
+'')''
FROM sys.sensitivity_classifications sc
JOIN sys.objects O
ON [sc].[major_id] = O.object_id
JOIN sys.columns C
ON [sc].[major_id] = C.object_id
AND [sc].[minor_id] = [C].[column_id]
--WHERE CAST(sc.[information_type] AS VARCHAR(500)) <> ''Other''
--WHERE [O].[name] LIKE ''[IIICommon_Bank_PTT_Master]''
ORDER BY CAST([sc].[information_type] AS VARCHAR(500)), [schema_name], [table_name], [column_name];
')
END
ELSE
BEGIN
SELECT
schema_name(O.schema_id) AS schema_name,
O.[name] AS table_name,
C.[name] AS column_name,
[EP].[information_type],
[EP].[sensitivity_label]
FROM
(
SELECT
IT.major_id,
IT.minor_id,
IT.information_type,
L.sensitivity_label
FROM
(
SELECT
major_id,
minor_id,
value AS information_type
FROM sys.extended_properties
WHERE NAME = 'sys_information_type_name'
) IT
FULL OUTER JOIN
(
SELECT
major_id,
minor_id,
value AS sensitivity_label
FROM sys.extended_properties
WHERE NAME = 'sys_sensitivity_label_name'
) L
ON IT.major_id = L.major_id AND IT.minor_id = L.minor_id
) EP
JOIN sys.objects O
ON EP.major_id = O.object_id
JOIN sys.columns C
ON EP.major_id = C.object_id AND EP.minor_id = C.column_id
;
END