Refreshing QA environment is not an easy work when your backup files are terabytes in size. The challenges are not only on space but also on time. Some tables are never envolved into QA phase. Restoring them will waste of your time and space. If you put tables in different file groups, then you would have the option to not restore them. If they are accessed by QA test, you can rename existing corrupted table (file groups are not restored) and create an empty table to make the logic correct. Now, if there is a huge table holding the historical data for the system, QA only need the latest 2 partitions of worth of data for testing, most of the DBAs in this case will restore entire database fist, switch useless partions out of the table, drop the table and file group. What if you have space limitation that only allow you restore part of the database? The solution would be

  • Restore database with primary file group and desired file groups
  • create table empty table which has the same structure as original, including the partition scheme
  • Swith the partition from existing table to the empty table
  • Rename tables

That’s a big work especially when you have hundreds of partitions across multiple tables. Here I post my script for that. It supports full, differential, log, and point in time retore, as well as multiple partitions over tables.

--- Step 1: open this script and connect it to the instance you are going to restore the backup
use master
set nocount on
declare @Execute bit; set @Execute = 0  -- this one is Zero, if it's one, the script will perform restoration
-- Step 2: Enter backup information 
declare @StopAt datetime
select @StopAt = '2011-09-30 11:39:01.850'
select @StopAt = null
if object_id('tempdb..#Backups') is not null
	drop table #Backups
create table #Backups (BackupID int identity(1,1) primary key, Type varchar(50) check(Type in ('Full','Diff', 'Log')), FilePath nvarchar(max) )
insert into #Backups(Type, FilePath) values('Full', 'c:\temp\test\test.bak')
/*insert into #Backups(Type, FilePath) values('Diff', 'c:\temp\test\diff.bak')
insert into #Backups(Type, FilePath) values('Log', 'c:\temp\test\Log.bak')
insert into #Backups(Type, FilePath) values('Log', 'c:\temp\test\Log1.bak')
*/
declare @Backup nvarchar(max); 
select @Backup = FilePath from #Backups where Type = 'Full'
if @Backup is null
begin
	raiserror('No full backup specified', 16, 1)
	return
end
-- Step 3: Endter the place where you want to put the data file, end with "\"
declare @TargetPath nvarchar(max); set @TargetPath = 'c:\temp\test\Target\'
-- Step 4: Enter the differential backup file name and 

-- Step 5: Enter the database you want to create, if the database exists. 
--         Important Note: If the target database exists, it will be DROPped
declare @TargetDatabaseName sysname set @TargetDatabaseName = 'TestTarget'
declare @DisabledLogicalName table (LogicalName sysname)
-- Step 6: Ensure @Execute is ZERO and Run this script
--         It will list the partitions. find the partition you don't want to restore
--         Copy past the sql generated below, you can run it again to verify
-- Step 7:  Change @Execute to ONE, then run
select @Execute = 1
---put unwanted filegroup here.....start
insert into @DisabledLogicalName values('F2010')
---put unwanted filegroup here.....end
if OBJECT_ID('tempdb..#FilesInBackup') is not null
	drop table #FilesInBackup
create table #FilesInBackup(LogicalName nvarchar(128), PhysicalName nvarchar(max), Type nvarchar(10), FileGroupName nvarchar(128), Size numeric(30,0), MaxSize numeric(30,0), FileID int, CreateLNS numeric(30,0), DropLSN numeric(30,0), UniqueID uniqueidentifier, ReadOnlyLSN numeric(30,0), readWriteLSN numeric(30,0), BackupSizeInBytes numeric(30,0), SourceBlockSize numeric(30,0), FileGroupID int, LogGroupGUID uniqueidentifier, DifferentialBaseLSN numeric(30,0), DifferentialBaseGUID uniqueidentifier, IsReadOnly bit, IsPresent bit, TDEThumbPrint uniqueidentifier)
insert into #FilesInBackup
	exec sp_executesql N'restore filelistonly from disk = @Backup', N'@Backup nvarchar(max)', @Backup
if @Execute = 0
begin
	;with x as (
		select 
				left(PhysicalName, len(PhysicalName) - charindex('\', reverse(PhysicalName))+1) Path, 
				* 
		from #FilesInBackup
	)
	select x.FileGroupName, x.LogicalName, replace(x.PhysicalName, Path, '') PhysicalFileName ,  case when y.LogicalName is null then '' else 'Excluded' end IsExcluded ,'insert into @DisabledLogicalName values('''+x.LogicalName+''')' SQL
	from x
		left outer join @DisabledLogicalName y on x.LogicalName = y.LogicalName
	where x.FileGroupName is not null
	order by x.FileGroupName, x.LogicalName
	return
end

delete x from #FilesInBackup x where exists(select 1 from @DisabledLogicalName y where x.LogicalName = y.LogicalName)
declare @SQL nvarchar(max); set @SQL = '
alter database ' + quotename(@TargetDatabaseName) + ' set single_user with rollback immediate;
drop database ' + quotename(@TargetDatabaseName)
if DB_ID(@TargetDatabaseName) is not null
	exec(@SQL)
select @SQL = 'use master
restore database ' + quotename(@TargetDatabaseName) + ' ' + stuff((select ', File = ''' + LogicalName + '''' from #FilesInBackup where FileGroupName is not null for xml path(''), type).value('.', 'nvarchar(max)'), 1, 1, '') +'
from disk = ''' + @Backup + '''
with partial, norecovery, stats = 1
' +(select '
	, move ''' + LogicalName + ''' to '''+@TargetPath+ replace(PhysicalName, left(PhysicalName, len(PhysicalName) - charindex('\', reverse(PhysicalName))+1), '') +''''  from #FilesInBackup for xml path(''), type).value('.', 'nvarchar(max)') +';'
exec (@SQL)	
-- differential backup
select @Backup = null
select @Backup = FilePath from #Backups where Type = 'Diff'
if(@Backup is not null)
begin
	select @SQL = 'restore database ' + quotename(@TargetDatabaseName) + '  from disk = ''' + @Backup + ''' with norecovery, stats = 1'
	exec (@SQL)
end

--log backup
declare c cursor static for
	select FilePath from #Backups where Type = 'Log' order by BackupID
open c
fetch next from c into @Backup
while @@fetch_status = 0
begin
	select @SQL = 'restore log ' + quotename(@TargetDatabaseName) + ' from disk = ''' + @Backup + ''' with norecovery, stats = 1'  + isnull(',stopat=''' + convert(varchar(100), @StopAt, 121) + '''','')
	print @SQL
	begin try
		exec(@SQL)
	end try
	begin catch
		print error_message()
	end catch
	fetch next from c into @Backup
end
close c 
deallocate c
---recovery
select @SQL = 'begin try
restore database ' + quotename(@TargetDatabaseName) + ' with recovery;
end try
begin catch
	print error_message()
end catch'
exec (@SQL)


declare @Proc nvarchar(218); set @Proc = quotename(@TargetDatabaseName)+'..sp_executesql'
if object_id('tempdb..#Partitions') is not null
	drop table #Partitions
create table #Partitions(
							ObjectID int, SchemaName sysname, TableName sysname, 
							IndexID int, IndexName sysname, FileGroupName sysname, 
							Status sysname, PartitionSchemeName sysname, PartitionFunctionName sysname, 
							RangeType sysname, PartitionNumber int, ParameterID int, 
							Value sql_variant, DataType sysname)
insert into #Partitions
exec @Proc N'

select distinct 
		i.object_id ObjectID, OBJECT_SCHEMA_NAME(i.object_id) as SchemaName, OBJECT_NAME(i.object_id) TableName, 
		i.index_id IndexID, i.name as IndexName, fg.name FileGroupName, 
		df.state_desc Status, ps.name PartitionSchemeName, pf.name PartitionFunctionName, 
		case when pf.boundary_value_on_right = 1 then ''Right'' else ''Left'' end RangeType, dds.destination_id PartitionNumber, rv.parameter_id,
		rv.Value as Value, t.name DataType
from sys.partition_schemes ps
	inner join sys.partition_functions pf on ps.function_id = pf.function_id
	inner join sys.data_spaces ds on ds.data_space_id = ps.data_space_id
	inner join sys.indexes i on i.data_space_id = ds.data_space_id
	inner join sys.destination_data_spaces dds on dds.partition_scheme_id = ps.data_space_id and dds.destination_id <= pf.fanout
	inner join sys.filegroups fg on fg.data_space_id = dds.data_space_id
	inner join sys.database_files df on df.data_space_id = fg.data_space_id
	inner join sys.partition_parameters pp ON pp.function_id = pf.function_id
	inner join sys.types t ON (t.user_type_id = pp.system_type_id ) or (t.system_type_id = pp.system_type_id and t.user_type_id = pp.user_type_id) and (t.is_user_defined = 0) and (t.is_assembly_type = 1)
	left outer join sys.partition_range_values rv on rv.function_id = pf.function_id  and rv.boundary_id = dds.destination_id
	order by 7, 3,4, 11
'

if not exists(select * from #Partitions where status <> 'ONLINE') -- all filegroups are restored
	return
declare @PartitionNameSuffix nvarchar(50); set @PartitionNameSuffix =  replace(cast(NEWID() as nvarchar(50)), '-', '')
if object_id('tempdb..#DestinationPartition') is not null
	drop table #DestinationPartition
;with x as
(
	select distinct 
			p1.ObjectID, p1.IndexID, p1.SchemaName, p1.TableName,
			p1.PartitionFunctionName, p1.PartitionNumber, p1.DataType , 
			p1.Value, p1.RangeType, PartitionSchemeName, 
			FileGroupName, p1.ParameterID
	from #Partitions p1
	where p1.Status = 'ONLINE'
		and exists(select 1 from #Partitions p2 where p2.PartitionFunctionName = p1.PartitionFunctionName and p2.Status <> 'ONLINE')
		and p1.IndexID in (0,1)
), y as
(
	select distinct PartitionFunctionName, RangeType, PartitionSchemeName, SchemaName, TableName
	from x
)
select 
	SchemaName, TableName, y.PartitionSchemeName+@PartitionNameSuffix NewPartitionSchemeName, 
	'create partition function ' + quotename(y.PartitionFunctionName+@PartitionNameSuffix) +
	'(' + stuff((
					select ',' + x.DataType 
					from x 
					where x.PartitionFunctionName = y.PartitionFunctionName 
						and x.PartitionSchemeName = y.PartitionSchemeName  
						and x.ParameterID is not null
					group by x.PartitionFunctionName, x.DataType, x.ParameterID 
					order by ParameterID 
					for xml path(''), type).value('.', 'nvarchar(max)'), 1, 1, '') +	')'+
	' as range ' + RangeType + ' for values (' +
	stuff((select distinct ',' + case when x.DataType in( 'int', 'bigint', 'decimal', 'numeric', 'float', 'real', 'bit', 'money', 'tinyint', 'binary', 'varbinary', 'smallint', 'smallmoney', 'timestamp', 'image') then convert(nvarchar(200), x.Value, 1)
							else '''' + convert(nvarchar(200), x.Value, 121) + ''''
						end from x 
			where x.PartitionFunctionName = y.PartitionFunctionName 
				and x.PartitionSchemeName = y.PartitionSchemeName  
				and x.Value is not null
			for xml path(''), type).value('.', 'nvarchar(max)'), 1, 1, '') +
	')' as PartitionFunctionDefinition,
	'create partition scheme ' + quotename(y.PartitionSchemeName+@PartitionNameSuffix) + 
	' as partition ' + quotename(y.PartitionFunctionName+@PartitionNameSuffix) + ' TO (' +
	stuff((
			select ',' + quotename(x.FileGroupName) 
			from x 
			where x.PartitionFunctionName = y.PartitionFunctionName 
				and x.PartitionSchemeName = y.PartitionSchemeName 
			order by x.PartitionNumber for xml path(''), type
			).value('.', 'nvarchar(max)'), 1, 1, '') +
	')' as PartitionSchemeDefinition,
	cast(null as nvarchar(max))TableDefinition
	into #DestinationPartition
from y

declare @TableName sysname, @SchemaName sysname, @FullTableName sysname, @Definition nvarchar(max), @NewPartitionSchemeName sysname
declare c cursor static for
	select SchemaName, TableName, NewPartitionSchemeName
	from #DestinationPartition
open c
fetch next from c into @SchemaName, @TableName, @NewPartitionSchemeName
while @@fetch_status = 0
begin
	select @FullTableName = @SchemaName + '.' + @TableName
	exec @Proc
N'declare @NewTableName sysname; set @NewTableName = @ExistingTableName + @PartitionNameSuffix
select @Definition = ''create table '' + @NewTableName + '' ('' +
(select  quotename(col.name) + '' '' +
case t.name 
				when ''binary'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name  in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
				when ''varbinary'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name  in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
				when ''datetime2'' then t.name + ''(''+cast(col.scale  as varchar(20))+'')''
                when ''decimal'' then t.name + ''(''+cast(col.precision as varchar(20))+'',''+cast(col.scale  as varchar(20))+'')''
                when ''numeric'' then t.name + ''(''+cast(col.precision as varchar(20))+'',''+cast(col.scale  as varchar(20))+'')''
                when ''char'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name  in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
                when ''nchar'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name  in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
                when ''nvarchar'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name  in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
                when ''varchar'' then t.name + case when col.max_length = -1 then ''(max)'' else ''('' + cast(case when t.name  in (''nvarchar'', ''nchar'') then col.max_length/2 else col.max_length end as varchar(20)) + '')'' end
                when ''datetimeoffset'' then t.name + ''(''+cast(col.scale  as varchar(20))+'')''
                else 
					t.name
		end + case when col.is_nullable = 1 then '' null'' else '' not null'' end + 
		ISNULL( (select '' default '' + dc.definition from sys.default_constraints dc where dc.parent_object_id = col.object_id and dc.parent_column_id = col.column_id ) ,'''')
		+'',''
from sys.columns col 
	inner join sys.types t on t.user_type_id = col.user_type_id
where col.object_id = object_id(@ExistingTableName)
order by col.column_id
for xml path(''''), type).value(''.'', ''nvarchar(max)'')+
'' Constraint '' + (select name+@PartitionNameSuffix from sys.indexes where object_id = object_id(@ExistingTableName) and index_id in(0,1)) + '' primary key clustered ('' + stuff((select '',''+quotename(col_name(ix.object_id, ix.column_id)) from sys.index_columns ix where ix.index_id in(0,1) and ix.object_id = object_id(@ExistingTableName) order by key_ordinal for xml path(''''), type).value(''.'', ''nvarchar(max)''), 1, 1, '''') +  
'') on '' + @NewParitionName + ''('' + 
stuff((select '',''+quotename(col_name(object_id, column_id)) from sys.index_columns where index_id in(0,1) and object_id = object_id(@ExistingTableName) and partition_ordinal >0  order by key_ordinal  for xml path(''''), type).value(''.'', ''nvarchar(max)''), 1, 1,'''') + ''));
'' + 
isnull((select ''create '' + case when i.is_unique = 1 then ''unique'' else ''''end + '' index '' + i.name +'' on '' + @NewTableName + '' ('' +
		stuff((select '',''+quotename(col_name(ix.object_id, ix.column_id)) from sys.index_columns ix where ix.index_id =i.index_id and ix.object_id = object_id(@ExistingTableName) and partition_ordinal =0 order by ix.key_ordinal for xml path(''''), type).value(''.'', ''nvarchar(max)''), 1, 1, '''') + 
		'')'' + '' on '' + @NewParitionName + ''('' + stuff((select '',''+quotename(col_name(ix.object_id, ix.column_id)) from sys.index_columns ix where ix.index_id = i.index_id and ix.object_id = object_id(@ExistingTableName) and ix.partition_ordinal >0  order by key_ordinal  for xml path(''''), type).value(''.'', ''nvarchar(max)''), 1, 1,'''')  + '');
''
from sys.indexes i 
where i.object_id = object_id(@ExistingTableName) 
	and i.index_id > 1
	and i.is_disabled = 0 
	and exists(	select 1 
				from sys.index_columns ix1 
				where ix1.index_id =i.index_id 
					and ix1.object_id = i.object_id 
					and ix1.partition_ordinal > 0 )
for xml path(''''), type).value(''.'', ''nvarchar(max)''), '''')', 
		N'@ExistingTableName sysname, @PartitionNameSuffix sysname, @NewParitionName sysname, @Definition nvarchar(max) output', @FullTableName, @PartitionNameSuffix, @NewPartitionSchemeName, @Definition output
	update #DestinationPartition set TableDefinition = @Definition where SchemaName = @SchemaName and TableName = @TableName
	fetch next from c into @SchemaName, @TableName, @NewPartitionSchemeName
end
close c
deallocate c

select @SQL = (
				select PartitionFunctionDefinition +'
' + PartitionSchemeDefinition + '
' + TableDefinition 
from #DestinationPartition
for xml path(''), type).value('.', 'nvarchar(max)')
print @sql
exec @Proc @SQL

---switch partition
declare @ExistingTableName sysname, @NewTableName sysname, @TempTableName sysname, @ProcRename sysname 
set @ProcRename = @TargetDatabaseName + '..sp_rename'
declare c cursor static for
	select quotename(SchemaName)+'.'+quotename(TableName), quotename(SchemaName)+'.'+quotename(TableName+ @PartitionNameSuffix), TableName
	from #DestinationPartition
open c
fetch next from c into @ExistingTableName, @NewTableName, @TableName
while @@fetch_status = 0
begin
	select @SQL = ';with x as (select rv.Value, dds.destination_id boundary_id, fg.name
	from sys.partition_schemes ps
	inner join sys.partition_functions pf on ps.function_id = pf.function_id
	inner join sys.data_spaces ds on ds.data_space_id = ps.data_space_id
	inner join sys.indexes i on i.data_space_id = ds.data_space_id
	inner join sys.destination_data_spaces dds on dds.partition_scheme_id = ps.data_space_id and dds.destination_id <= pf.fanout 
	inner join sys.filegroups fg on fg.data_space_id = dds.data_space_id
	left join sys.partition_range_values rv on rv.function_id = pf.function_id  and rv.boundary_id = dds.destination_id
where i.object_id = object_id(@ExistingTableName) and i.index_id in(0,1)
), y as (
select distinct rv.Value, dds.destination_id boundary_id, fg.name
from sys.partition_schemes ps
	inner join sys.partition_functions pf on ps.function_id = pf.function_id
	inner join sys.data_spaces ds on ds.data_space_id = ps.data_space_id
	inner join sys.indexes i on i.data_space_id = ds.data_space_id
	 
	inner join sys.destination_data_spaces dds on dds.partition_scheme_id = ps.data_space_id and dds.destination_id <= pf.fanout
	inner join sys.filegroups fg on fg.data_space_id = dds.data_space_id
	left join sys.partition_range_values rv on rv.function_id = pf.function_id  and rv.boundary_id = dds.destination_id
where i.object_id = object_id(@NewTableName) and i.index_id in(0,1)
)
select @SQL = (select ''alter table ''+@ExistingTableName+'' switch partition ''+cast(x.boundary_id as varchar(20))+'' TO ''+@NewTableName+'' partition ''+cast(y.boundary_id as varchar(20))+'' ;''
from x
	inner join y on (x.Value is null and y.Value is null or x.Value = y.Value)
for xml path(''''), type).value(''.'', ''nvarchar(max)'')

'
	exec @Proc @SQL, N'@ExistingTableName sysname, @NewTableName sysname, @SQL nvarchar(max) output', @ExistingTableName, @NewTableName, @SQL output
	--select @sql
	exec @Proc @SQL
-- rename
	select @TempTableName = @TableName + @PartitionNameSuffix + 'temp'
	exec @ProcRename @ExistingTableName, @TempTableName
	exec @ProcRename @NewTableName, @TableName
	fetch next from c into @ExistingTableName, @NewTableName, @TableName
end
close c
deallocate c

There are few parameters; they are highlighted.

  • Line 7 and 8, setup the point in time value as needed. This applies when you need to restore the transaction logs
  • Line 12 ~ 16, enter the files needed to be restored, type can be Full, Diff, and Log
  • Line 36, if @Execute = 0, the script will not perform any restoration. It will list all the files and file groups in the backup instead. The last column of the list shows SQLs that you can copy and past the them to line 38 to let the script to not restore the files. After pasting the SQL, you can run tihs script with @Execute = 0 again to ensure the files marked as "Excluded" are the files unwanted. When you turn @Execute = 1, the script will do the restoration.
  • Line 38, Specify the database files you don't want to restore.

After having above parameters specified, you can turn @Execute = 1 to perform the restoration. Now let's have a test below.
Create test database including table dbo.test with few partitions

USE master
set nocount on
GO
if DB_ID('test') is not null
begin
	alter database test set single_user with rollback immediate
	drop database test
end
go
create database test on primary ( NAME = N'test', FILENAME = N'c:\temp\test\test.mdf' ),
filegroup [FG-2008]( NAME = N'F2008', FILENAME = N'c:\temp\test\F2008.ndf'),
filegroup [FG-2009]( NAME = N'F2009', FILENAME = N'c:\temp\test\F2009.ndf'),
filegroup [FG-2010]( NAME = N'F2010', FILENAME = N'c:\temp\test\F2010.ndf'),
filegroup [FG-2011]( NAME = N'F2011', FILENAME = N'c:\temp\test\F2011.ndf'),
filegroup [FG-NextUse]( NAME = N'FNextUse', FILENAME = N'c:\temp\test\FNextUse.ndf')
log on ( NAME = N'test_log', FILENAME = N'c:\temp\test\test_log.LDF')
go
USE [test]
go
create partition function [PF_Test](int) as range right for values (2009, 2010, 2011)
create partition scheme [PS_Test] as partition [PF_Test] to ([FG-2008], [FG-2009], [FG-2010], [FG-2011])
go
create table dbo.Test(
	PartitionKey int,
	Key1 int,
	constraint PK_Test primary key(PartitionKey,Key1) ON PS_Test(PartitionKey)
) ON PS_Test(PartitionKey)
go
--Generate some data
declare @i int, @j int
select @i = 2008
while @i<=2011
begin
	select @j = 0
	while @j<10
	begin
		insert into dbo.Test (PartitionKey, Key1) values(@i, 1)
		select @j=@j+1
	end
 select @i=@i+1
end
backup database test to disk= 'c:\temp\test.bak' with init
select object_name(object_id), partition_number, rows
from sys.partitions
where object_id = object_id('test')

Now the database and back are created. The table has 40 rows.

Processed 160 pages for database 'test', file 'test' on file 1.
Processed 16 pages for database 'test', file 'F2008' on file 1.
Processed 16 pages for database 'test', file 'F2009' on file 1.
Processed 16 pages for database 'test', file 'F2010' on file 1.
Processed 16 pages for database 'test', file 'F2011' on file 1.
Processed 8 pages for database 'test', file 'FNextUse' on file 1.
Processed 1 pages for database 'test', file 'test_log' on file 1.
BACKUP DATABASE successfully processed 233 pages in 0.083 seconds (21.849 MB/sec).
name       partition_number rows
---------- ---------------- --------------------
Test       1                10
Test       2                10
Test       3                10
Test       4                10

Now let's run the script to take the data for 2010 out. Then check the result:

use TestTarget
select cast(Object_name(object_id) as varchar(10)) name, partition_number, rows
from sys.partitions where object_id = object_id('dbo.Test')
select PartitionKey, COUNT(*) Count from test group by PartitionKey
name partition_number rows
 ---------- ---------------- --------------------
 Test 1 10
 Test 2 10
 Test 3 10
PartitionKey Count
 ------------ -----------
 2008 10
 2009 10
 2011 10
Restore Database With Partitions Excluded (or Selected Partitions)

You May Also Like

Leave a Reply

Your email address will not be published. Required fields are marked *

C# | HTML | Plain Text | SQL | XHTML | XML | XSLT |

This site uses Akismet to reduce spam. Learn how your comment data is processed.