我需要从SqlDataReader中批量加载DataTable。 SqlDataReader将返回数百万条记录,并且DataTable Load()方法将耗尽可用内存。使用Powershell批量加载SQLDataReader中的DataTable
这里是我当前的代码:
[cmdletBinding(DefaultParameterSetName = 'Instance',
SupportsShouldProcess = $true,
ConfirmImpact = 'High')]
Param (
[string] $SrcServer = "MySQLServer,12345",
[string] $SrcDatabase = "SourceDb",
[string] $SrcTable = "dbo.SourceTable",
[string] $SrcQuery = "SELECT TOP 100 HASHBYTES('SHA',stay_number) as stay_number_h, * FROM $SrcTable",
[string] $TgtServer,
[string] $TgtDatabase = "TargetDb",
[string] $TgtTable = "tmp.TargetTable",
[switch] $Truncate = $true
)
Function ConnectionString([string] $ServerName, [string] $DbName)
{
"Data Source=$ServerName;Initial Catalog=$DbName;Integrated Security=True;Connection Timeout=30"
}
########## Main body ############
Write-Host "Starting..."
If ($TgtServer.Length –eq 0) {
$TgtServer = $SrcServer
}
If ($TgtDatabase.Length –eq 0) {
$TgtDatabase = $SrcDatabase
}
If ($TgtTable.Length –eq 0) {
$TgtTable = $SrcTable
}
If ($Truncate) {
Write-Host "Truncating $TgtTable"
$TruncateSql = "TRUNCATE TABLE " + $TgtTable
Sqlcmd -S $TgtServer -d $TgtDatabase -Q $TruncateSql
}
$SrcConnStr = ConnectionString $SrcServer $SrcDatabase
$SrcConn = New-Object System.Data.SqlClient.SQLConnection($SrcConnStr)
$CmdText = $SrcQuery
$SqlCommand = New-Object system.Data.SqlClient.SqlCommand($CmdText, $SrcConn)
$SrcConn.Open()
[System.Data.SqlClient.SqlDataReader] $SqlReader = $SqlCommand.ExecuteReader()
# Can we convert the SqlReader to a DataTable?
$dtSchema = $SqlReader.GetSchemaTable()
$dt = New-Object System.Data.DataTable
if ($dtSchema -ne $null)
{
foreach ($drow in $dtSchema.Rows)
{
$columnName = $drow["ColumnName"]
$column = New-Object System.Data.DataColumn($columnName, $drow["DataType"])
$column.Unique = $drow["IsUnique"]
$column.AllowDBNull = $drow["AllowDBNull"]
$column.AutoIncrement = $drow["IsAutoIncrement"]
$dt.Columns.Add($column)
}
}
Write-Host "Now loading DataTable"
for ($i=0;$i -le 10; $i++) {
$i
$null = $dt.LoadDataRow($SqlReader,$true)
}
Write-Host "DataTable filled, how long and check memory consumption!"
sleep 30
$datatable.Clear()
Write-Host "Finished"
相关链接:
CSV到SQL Server:
https://gallery.technet.microsoft.com/scriptcenter/Import-Large-CSVs-into-SQL-216223d9 https://gallery.technet.microsoft.com/scriptcenter/Import-Large-CSVs-into-SQL-fa339046
SQL服务器到SQL Server:
https://blogs.technet.microsoft.com/heyscriptingguy/2011/05/06/use-powershell-to-copy-a-table-between-two-sql-server-instances/ https://newsqlblog.com/2011/08/12/moving-data-between-sql-servers-with-powershell/ https://raw.githubusercontent.com/RamblingCookieMonster/PowerShell/master/Invoke-SQLBulkCopy.ps1
理想的情况下我的最终解决方案将支持SS和外部文件的导入(实际上任何可以实现为一个DataTable)