Refreshing QA environment is not an easy work when your backup files are terabytes in size. The challenges are not only on space but also on time. Some tables are never envolved into QA phase. Restoring them will waste of your time and space. If you put tables in different file groups, then you would have the option to not restore them. If they are accessed by QA test, you can rename existing corrupted table (file groups are not restored) and create an empty table to make the logic correct. Now, if there is a huge table holding the historical data for the system, QA only need the latest 2 partitions of worth of data for testing, most of the DBAs in this case will restore entire database fist, switch useless partions out of the table, drop the table and file group. What if you have space limitation that only allow you restore part of the database? The solution would be
- Restore database with primary file group and desired file groups
- create table empty table which has the same structure as original, including the partition scheme
- Swith the partition from existing table to the empty table
- Rename tables
That’s a big work especially when you have hundreds of partitions across multiple tables. Here I post my script for that. It supports full, differential, log, and point in time retore, as well as multiple partitions over tables.
--- Step 1: open this script and connect it to the instance you are going to restore the backup
use master
set nocount on
declare @Execute bit; set @Execute = 0 -- this one is Zero, if it's one, the script will perform restoration
-- Step 2: Enter backup information
declare @StopAt datetime
select @StopAt = '2011-09-30 11:39:01.850'
select @StopAt = null
if object_id('tempdb..#Backups') is not null
drop table #Backups
create table #Backups (BackupID int identity(1,1) primary key, Type varchar(50) check(Type in ('Full','Diff', 'Log')), FilePath nvarchar(max) )
insert into #Backups(Type, FilePath) values('Full', 'c:\temp\test\test.bak')
/*insert into #Backups(Type, FilePath) values('Diff', 'c:\temp\test\diff.bak')
insert into #Backups(Type, FilePath) values('Log', 'c:\temp\test\Log.bak')
insert into #Backups(Type, FilePath) values('Log', 'c:\temp\test\Log1.bak')
*/
declare @Backup nvarchar(max);
select @Backup = FilePath from #Backups where Type = 'Full'
if @Backup is null
begin
raiserror('No full backup specified', 16, 1)
return
end
-- Step 3: Endter the place where you want to put the data file, end with "\"
declare @TargetPath nvarchar(max); set @TargetPath = 'c:\temp\test\Target\'
-- Step 4: Enter the differential backup file name and
-- Step 5: Enter the database you want to create, if the database exists.
-- Important Note: If the target database exists, it will be DROPped
declare @TargetDatabaseName sysname set @TargetDatabaseName = 'TestTarget'
declare @DisabledLogicalName table (LogicalName sysname)
-- Step 6: Ensure @Execute is ZERO and Run this script
-- It will list the partitions. find the partition you don't want to restore
-- Copy past the sql generated below, you can run it again to verify
-- Step 7: Change @Execute to ONE, then run
select @Execute = 1
---put unwanted filegroup here.....start
insert into @DisabledLogicalName values('F2010')
---put unwanted filegroup here.....end
if OBJECT_ID('tempdb..#FilesInBackup') is not null
drop table #FilesInBackup
create table #FilesInBackup(LogicalName nvarchar(128), PhysicalName nvarchar(max), Type nvarchar(10), FileGroupName nvarchar(128), Size numeric(30,0), MaxSize numeric(30,0), FileID int, CreateLNS numeric(30,0), DropLSN numeric(30,0), UniqueID uniqueidentifier, ReadOnlyLSN numeric(30,0), readWriteLSN numeric(30,0), BackupSizeInBytes numeric(30,0), SourceBlockSize numeric(30,0), FileGroupID int, LogGroupGUID uniqueidentifier, DifferentialBaseLSN numeric(30,0), DifferentialBaseGUID uniqueidentifier, IsReadOnly bit, IsPresent bit, TDEThumbPrint uniqueidentifier)
insert into #FilesInBackup
exec sp_executesql N'restore filelistonly from disk = @Backup', N'@Backup nvarchar(max)', @Backup
if @Execute = 0
begin
;with x as (
select
left(PhysicalName, len(PhysicalName) - charindex('\', reverse(PhysicalName))+1) Path,
*
from #FilesInBackup
)
select x.FileGroupName, x.LogicalName, replace(x.PhysicalName, Path, '') PhysicalFileName , case when y.LogicalName is null then '' else 'Excluded' end IsExcluded ,'insert into @DisabledLogicalName values('''+x.LogicalName+''')' SQL
from x
left outer join @DisabledLogicalName y on x.LogicalName = y.LogicalName
where x.FileGroupName is not null
order by x.FileGroupName, x.LogicalName
return
end
delete x from #FilesInBackup x where exists(select 1 from @DisabledLogicalName y where x.LogicalName = y.LogicalName)
declare @SQL nvarchar(max); set @SQL = '
alter database ' + quotename(@TargetDatabaseName) + ' set single_user with rollback immediate;
drop database ' + quotename(@TargetDatabaseName)
if DB_ID(@TargetDatabaseName) is not null
exec(@SQL)
select @SQL = 'use master
restore database ' + quotename(@TargetDatabaseName) + ' ' + stuff((select ', File = ''' + LogicalName + '''' from #FilesInBackup where FileGroupName is not null for xml path(''), type).value('.', 'nvarchar(max)'), 1, 1, '') +'
from disk = ''' + @Backup + '''
with partial, norecovery, stats = 1
' +(select '
, move ''' + LogicalName + ''' to '''+@TargetPath+ replace(PhysicalName, left(PhysicalName, len(PhysicalName) - charindex('\', reverse(PhysicalName))+1), '') +'''' from #FilesInBackup for xml path(''), type).value('.', 'nvarchar(max)') +';'
exec (@SQL)
-- differential backup
select @Backup = null
select @Backup = FilePath from #Backups where Type = 'Diff'
if(@Backup is not null)
begin
select @SQL = 'restore database ' + quotename(@TargetDatabaseName) + ' from disk = ''' + @Backup + ''' with norecovery, stats = 1'
exec (@SQL)
end
--log backup
declare c cursor static for
select FilePath from #Backups where Type = 'Log' order by BackupID
open c
fetch next from c into @Backup
while @@fetch_status = 0
begin
select @SQL = 'restore log ' + quotename(@TargetDatabaseName) + ' from disk = ''' + @Backup + ''' with norecovery, stats = 1' + isnull(',stopat=''' + convert(varchar(100), @StopAt, 121) + '''','')
print @SQL
begin try
exec(@SQL)
end try
begin catch
print error_message()
end catch
fetch next from c into @Backup
end
close c
deallocate c
---recovery
select @SQL = 'begin try
restore database ' + quotename(@TargetDatabaseName) + ' with recovery;
end try
begin catch
print error_message()
end catch'
exec (@SQL)
declare @Proc nvarchar(218); set @Proc = quotename(@TargetDatabaseName)+'..sp_executesql'
if object_id('tempdb..#Partitions') is not null
drop table #Partitions
create table #Partitions(
ObjectID int, SchemaName sysname, TableName sysname,
IndexID int, IndexName sysname, FileGroupName sysname,
Status sysname, PartitionSchemeName sysname, PartitionFunctionName sysname,
RangeType sysname, PartitionNumber int, ParameterID int,
Value sql_variant, DataType sysname)
insert into #Partitions
exec @Proc N'
select distinct
i.object_id ObjectID, OBJECT_SCHEMA_NAME(i.object_id) as SchemaName, OBJECT_NAME(i.object_id) TableName,
i.index_id IndexID, i.name as IndexName, fg.name FileGroupName,
df.state_desc Status, ps.name PartitionSchemeName, pf.name PartitionFunctionName,
case when pf.boundary_value_on_right = 1 then ''Right'' else ''Left'' end RangeType, dds.destination_id PartitionNumber, rv.parameter_id,
rv.Value as Value, t.name DataType
from sys.partition_schemes ps
inner join sys.partition_functions pf on ps.function_id = pf.function_id
inner join sys.data_spaces ds on ds.data_space_id = ps.data_space_id
inner join sys.indexes i on i.data_space_id = ds.data_space_id
inner join sys.destination_data_spaces dds on dds.partition_scheme_id = ps.data_space_id and dds.destination_id <= pf.fanout
inner join sys.filegroups fg on fg.data_space_id = dds.data_space_id
inner join sys.database_files df on df.data_space_id = fg.data_space_id
inner join sys.partition_parameters pp ON pp.function_id = pf.function_id
inner join sys.types t ON (t.user_type_id = pp.system_type_id ) or (t.system_type_id = pp.system_type_id and t.user_type_id = pp.user_type_id) and (t.is_user_defined = 0) and (t.is_assembly_type = 1)
left outer join sys.partition_range_values rv on rv.function_id = pf.function_id and rv.boundary_id = dds.destination_id
order by 7, 3,4, 11
'
if not exists(select * from #Partitions where status <> 'ONLINE') -- all filegroups are restored
return
declare @PartitionNameSuffix nvarchar(50); set @PartitionNameSuffix = replace(cast(NEWID() as nvarchar(50)), '-', '')
if object_id('tempdb..#DestinationPartition') is not null
drop table #DestinationPartition
;with x as
(
select distinct
p1.ObjectID, p1.IndexID, p1.SchemaName, p1.TableName,
p1.PartitionFunctionName, p1.PartitionNumber, p1.DataType ,
p1.Value, p1.RangeType, PartitionSchemeName,
FileGroupName, p1.ParameterID
from #Partitions p1
where p1.Status = 'ONLINE'
and exists(select 1 from #Partitions p2 where p2.PartitionFunctionName = p1.PartitionFunctionName and p2.Status <> 'ONLINE')
and p1.IndexID in (0,1)
), y as
(
select distinct PartitionFunctionName, RangeType, PartitionSchemeName, SchemaName, TableName
from x
)
select
SchemaName, TableName, y.PartitionSchemeName+@PartitionNameSuffix NewPartitionSchemeName,
'create partition function ' + quotename(y.PartitionFunctionName+@PartitionNameSuffix) +
'(' + stuff((
select ',' + x.DataType
from x
where x.PartitionFunctionName = y.PartitionFunctionName
and x.PartitionSchemeName = y.PartitionSchemeName
and x.ParameterID is not null
group by x.PartitionFunctionName, x.DataType, x.ParameterID
order by ParameterID
for xml path(''), type).value('.', 'nvarchar(max)'), 1, 1, '') + ')'+
' as range ' + RangeType + ' for values (' +
stuff((select distinct ',' + case when x.DataType in( 'int', 'bigint', 'decimal', 'numeric', 'float', 'real', 'bit', 'money', 'tinyint', 'binary', 'varbinary', 'smallint', 'smallmoney', 'timestamp', 'image') then convert(nvarchar(200), x.Value, 1)
else '''' + convert(nvarchar(200), x.Value, 121) + ''''
end from x
where x.PartitionFunctionName = y.PartitionFunctionName
and x.PartitionSchemeName = y.PartitionSchemeName
and x.Value is not null
for xml path(''), type).value('.', 'nvarchar(max)'), 1, 1, '') +
')' as PartitionFunctionDefinition,
'create partition scheme ' + quotename(y.PartitionSchemeName+@PartitionNameSuffix) +
' as partition ' + quotename(y.PartitionFunctionName+@PartitionNameSuffix) + ' TO (' +
stuff((
select ',' + quotename(x.FileGroupName)
from x
where x.PartitionFunctionName = y.PartitionFunctionName
and x.PartitionSchemeName = y.PartitionSchemeName
order by x.PartitionNumber for xml path(''), type
).value('.', 'nvarchar(max)'), 1, 1, '') +
')' as PartitionSchemeDefinition,
cast(null as nvarchar(max))TableDefinition
into #DestinationPartition
from y
declare @TableName sysname, @SchemaName sysname, @FullTableName sysname, @Definition nvarchar(max), @NewPartitionSchemeName sysname
declare c cursor static for
select SchemaName, TableName, NewPartitionSchemeName
from #DestinationPartition
open c
fetch next from c into @SchemaName, @TableName, @NewPartitionSchemeName
while @@fetch_status = 0
begin
select @FullTableName = @SchemaName + '.' + @TableName
exec @Proc
N'declare @NewTableName sysname; set @NewTableName = @ExistingTableName + @PartitionNameSuffix
select @Definition = ''create table '' + @NewTableName + '' ('' +
(select quotename(col.name) + '' '' +
case t.name
when ''binary'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
when ''varbinary'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
when ''datetime2'' then t.name + ''(''+cast(col.scale as varchar(20))+'')''
when ''decimal'' then t.name + ''(''+cast(col.precision as varchar(20))+'',''+cast(col.scale as varchar(20))+'')''
when ''numeric'' then t.name + ''(''+cast(col.precision as varchar(20))+'',''+cast(col.scale as varchar(20))+'')''
when ''char'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
when ''nchar'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
when ''nvarchar'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
when ''varchar'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
when ''datetimeoffset'' then t.name + ''(''+cast(col.scale as varchar(20))+'')''
else
t.name
end + case when col.is_nullable = 1 then '' null'' else '' not null'' end +
ISNULL( (select '' default '' + dc.definition from sys.default_constraints dc where dc.parent_object_id = col.object_id and dc.parent_column_id = col.column_id ) ,'''')
+'',''
from sys.columns col
inner join sys.types t on t.user_type_id = col.user_type_id
where col.object_id = object_id(@ExistingTableName)
order by col.column_id
for xml path(''''), type).value(''.'', ''nvarchar(max)'')+
'' Constraint '' + (select name+@PartitionNameSuffix from sys.indexes where object_id = object_id(@ExistingTableName) and index_id in(0,1)) + '' primary key clustered ('' + stuff((select '',''+quotename(col_name(ix.object_id, ix.column_id)) from sys.index_columns ix where ix.index_id in(0,1) and ix.object_id = object_id(@ExistingTableName) order by key_ordinal for xml path(''''), type).value(''.'', ''nvarchar(max)''), 1, 1, '''') +
'') on '' + @NewParitionName + ''('' +
stuff((select '',''+quotename(col_name(object_id, column_id)) from sys.index_columns where index_id in(0,1) and object_id = object_id(@ExistingTableName) and partition_ordinal >0 order by key_ordinal for xml path(''''), type).value(''.'', ''nvarchar(max)''), 1, 1,'''') + ''));
'' +
isnull((select ''create '' + case when i.is_unique = 1 then ''unique'' else ''''end + '' index '' + i.name +'' on '' + @NewTableName + '' ('' +
stuff((select '',''+quotename(col_name(ix.object_id, ix.column_id)) from sys.index_columns ix where ix.index_id =i.index_id and ix.object_id = object_id(@ExistingTableName) and partition_ordinal =0 order by ix.key_ordinal for xml path(''''), type).value(''.'', ''nvarchar(max)''), 1, 1, '''') +
'')'' + '' on '' + @NewParitionName + ''('' + stuff((select '',''+quotename(col_name(ix.object_id, ix.column_id)) from sys.index_columns ix where ix.index_id = i.index_id and ix.object_id = object_id(@ExistingTableName) and ix.partition_ordinal >0 order by key_ordinal for xml path(''''), type).value(''.'', ''nvarchar(max)''), 1, 1,'''') + '');
''
from sys.indexes i
where i.object_id = object_id(@ExistingTableName)
and i.index_id > 1
and i.is_disabled = 0
and exists( select 1
from sys.index_columns ix1
where ix1.index_id =i.index_id
and ix1.object_id = i.object_id
and ix1.partition_ordinal > 0 )
for xml path(''''), type).value(''.'', ''nvarchar(max)''), '''')',
N'@ExistingTableName sysname, @PartitionNameSuffix sysname, @NewParitionName sysname, @Definition nvarchar(max) output', @FullTableName, @PartitionNameSuffix, @NewPartitionSchemeName, @Definition output
update #DestinationPartition set TableDefinition = @Definition where SchemaName = @SchemaName and TableName = @TableName
fetch next from c into @SchemaName, @TableName, @NewPartitionSchemeName
end
close c
deallocate c
select @SQL = (
select PartitionFunctionDefinition +'
' + PartitionSchemeDefinition + '
' + TableDefinition
from #DestinationPartition
for xml path(''), type).value('.', 'nvarchar(max)')
print @sql
exec @Proc @SQL
---switch partition
declare @ExistingTableName sysname, @NewTableName sysname, @TempTableName sysname, @ProcRename sysname
set @ProcRename = @TargetDatabaseName + '..sp_rename'
declare c cursor static for
select quotename(SchemaName)+'.'+quotename(TableName), quotename(SchemaName)+'.'+quotename(TableName+ @PartitionNameSuffix), TableName
from #DestinationPartition
open c
fetch next from c into @ExistingTableName, @NewTableName, @TableName
while @@fetch_status = 0
begin
select @SQL = ';with x as (select rv.Value, dds.destination_id boundary_id, fg.name
from sys.partition_schemes ps
inner join sys.partition_functions pf on ps.function_id = pf.function_id
inner join sys.data_spaces ds on ds.data_space_id = ps.data_space_id
inner join sys.indexes i on i.data_space_id = ds.data_space_id
inner join sys.destination_data_spaces dds on dds.partition_scheme_id = ps.data_space_id and dds.destination_id <= pf.fanout
inner join sys.filegroups fg on fg.data_space_id = dds.data_space_id
left join sys.partition_range_values rv on rv.function_id = pf.function_id and rv.boundary_id = dds.destination_id
where i.object_id = object_id(@ExistingTableName) and i.index_id in(0,1)
), y as (
select distinct rv.Value, dds.destination_id boundary_id, fg.name
from sys.partition_schemes ps
inner join sys.partition_functions pf on ps.function_id = pf.function_id
inner join sys.data_spaces ds on ds.data_space_id = ps.data_space_id
inner join sys.indexes i on i.data_space_id = ds.data_space_id
inner join sys.destination_data_spaces dds on dds.partition_scheme_id = ps.data_space_id and dds.destination_id <= pf.fanout
inner join sys.filegroups fg on fg.data_space_id = dds.data_space_id
left join sys.partition_range_values rv on rv.function_id = pf.function_id and rv.boundary_id = dds.destination_id
where i.object_id = object_id(@NewTableName) and i.index_id in(0,1)
)
select @SQL = (select ''alter table ''+@ExistingTableName+'' switch partition ''+cast(x.boundary_id as varchar(20))+'' TO ''+@NewTableName+'' partition ''+cast(y.boundary_id as varchar(20))+'' ;''
from x
inner join y on (x.Value is null and y.Value is null or x.Value = y.Value)
for xml path(''''), type).value(''.'', ''nvarchar(max)'')
'
exec @Proc @SQL, N'@ExistingTableName sysname, @NewTableName sysname, @SQL nvarchar(max) output', @ExistingTableName, @NewTableName, @SQL output
--select @sql
exec @Proc @SQL
-- rename
select @TempTableName = @TableName + @PartitionNameSuffix + 'temp'
exec @ProcRename @ExistingTableName, @TempTableName
exec @ProcRename @NewTableName, @TableName
fetch next from c into @ExistingTableName, @NewTableName, @TableName
end
close c
deallocate c
There are few parameters; they are highlighted.
- Line 7 and 8, setup the point in time value as needed. This applies when you need to restore the transaction logs
- Line 12 ~ 16, enter the files needed to be restored, type can be Full, Diff, and Log
- Line 36, if @Execute = 0, the script will not perform any restoration. It will list all the files and file groups in the backup instead. The last column of the list shows SQLs that you can copy and past the them to line 38 to let the script to not restore the files. After pasting the SQL, you can run tihs script with @Execute = 0 again to ensure the files marked as "Excluded" are the files unwanted. When you turn @Execute = 1, the script will do the restoration.
- Line 38, Specify the database files you don't want to restore.
After having above parameters specified, you can turn @Execute = 1 to perform the restoration. Now let's have a test below.
Create test database including table dbo.test with few partitions
USE master
set nocount on
GO
if DB_ID('test') is not null
begin
alter database test set single_user with rollback immediate
drop database test
end
go
create database test on primary ( NAME = N'test', FILENAME = N'c:\temp\test\test.mdf' ),
filegroup [FG-2008]( NAME = N'F2008', FILENAME = N'c:\temp\test\F2008.ndf'),
filegroup [FG-2009]( NAME = N'F2009', FILENAME = N'c:\temp\test\F2009.ndf'),
filegroup [FG-2010]( NAME = N'F2010', FILENAME = N'c:\temp\test\F2010.ndf'),
filegroup [FG-2011]( NAME = N'F2011', FILENAME = N'c:\temp\test\F2011.ndf'),
filegroup [FG-NextUse]( NAME = N'FNextUse', FILENAME = N'c:\temp\test\FNextUse.ndf')
log on ( NAME = N'test_log', FILENAME = N'c:\temp\test\test_log.LDF')
go
USE [test]
go
create partition function [PF_Test](int) as range right for values (2009, 2010, 2011)
create partition scheme [PS_Test] as partition [PF_Test] to ([FG-2008], [FG-2009], [FG-2010], [FG-2011])
go
create table dbo.Test(
PartitionKey int,
Key1 int,
constraint PK_Test primary key(PartitionKey,Key1) ON PS_Test(PartitionKey)
) ON PS_Test(PartitionKey)
go
--Generate some data
declare @i int, @j int
select @i = 2008
while @i<=2011
begin
select @j = 0
while @j<10
begin
insert into dbo.Test (PartitionKey, Key1) values(@i, 1)
select @j=@j+1
end
select @i=@i+1
end
backup database test to disk= 'c:\temp\test.bak' with init
select object_name(object_id), partition_number, rows
from sys.partitions
where object_id = object_id('test')
Now the database and back are created. The table has 40 rows.
Processed 160 pages for database 'test', file 'test' on file 1. Processed 16 pages for database 'test', file 'F2008' on file 1. Processed 16 pages for database 'test', file 'F2009' on file 1. Processed 16 pages for database 'test', file 'F2010' on file 1. Processed 16 pages for database 'test', file 'F2011' on file 1. Processed 8 pages for database 'test', file 'FNextUse' on file 1. Processed 1 pages for database 'test', file 'test_log' on file 1. BACKUP DATABASE successfully processed 233 pages in 0.083 seconds (21.849 MB/sec). name partition_number rows ---------- ---------------- -------------------- Test 1 10 Test 2 10 Test 3 10 Test 4 10
Now let's run the script to take the data for 2010 out. Then check the result:
use TestTarget
select cast(Object_name(object_id) as varchar(10)) name, partition_number, rows
from sys.partitions where object_id = object_id('dbo.Test')
select PartitionKey, COUNT(*) Count from test group by PartitionKey
name partition_number rows ---------- ---------------- -------------------- Test 1 10 Test 2 10 Test 3 10 PartitionKey Count ------------ ----------- 2008 10 2009 10 2011 10