Is there a more efficient way to filter thousands of items from a SharePoint library than this below?
$FilteredListItems=$list.Items | ?{$_["CustNum"] -eq$currentCustNumber}
Scenario:
• I am exporting select files from a SharePoint library that has over 60,000 documents in it
• For each customer number, I filter the queried library items on that number
• I then iterate through each of the filtered items and export selected files
Working Code:
Below is the working code, but the slowest block of code is the filtering noted above that takes about 17 seconds each time. That would be almost 12 hours of processing for the 2500 customer numbers I’ll be filtering on.
# ExportFilesFromSharePoint
# Author: King Elder
# PLAN:
# - Iterates through CSV having customer numbers (NetSuite ID's)
# - Query SharePoint Library and filter on NetSuite ID
# - Loop through files to find Contracts or Addendums
# - Rename file if Customer Number not already in name
# - Export file to file system and write data to CSV file
####################### Variables ########################
$ExportedFileDestination="C:\\STAGING\\20140212ExportedFiles\\DocumentLibrary"
$csvMappingFile="C:\\STAGING\\20140212ExportedFiles\\SharePointExport.csv"
$SPWebUrl="http://SharePointSite"# <Url of the specific site>
$SPLibraryUrl="http://SharePointSite/DocumentLibrary/Documents/"
$CsvCustomerDataPathToFile="C:\\STAGING\\20140212ExportedFiles\\CustomerAccountNumbers.csv"
###########################################################
# Create mapping CSV with headers if doesn't exists
if (!(Test-Path-path$csvMappingFile))
{
"CustomerNumber, FileName">> $csvMappingFile
}
$web=Get-SPWeb-Identity$SPWebUrl
$list=$web.GetList($SPLibraryUrl)
$mpiData=Import-Csv-path$CsvCustomerDataPathToFile
Foreach ($CsvItemin$mpiData) {
$currentCustNumber=$CsvItem.'Customer Number'
# Filter the SharePoint list on the customer number (takes time since 60,361 items)
$FilteredListItems=$list.Items | ?{$_["CustNum"] -eq$currentCustNumber}
foreach($fileItemin$FilteredListItems)
{
if($fileItem.Name-Like"*addendum*"-or$fileItem.Name-Like"*contract*")
{
# Add Customer number if not in file name
if($fileItem.Name-Like"*$currentCustNumber*")
{
$newFileName=$fileItem.Name
}
else
{
$newFileName=$currentCustNumber+"."+$fileItem.Name
}
$newFilePath=$ExportedFileDestination+"\\"+$newFileName
$tempSourceFile="Documents/"+$fileItem.Name
# Download file
try {
$binary=$web.GetFile($tempSourceFile).OpenBinary()
$stream=New-ObjectSystem.IO.FileStream($newFilePath), Create
$writer=New-ObjectSystem.IO.BinaryWriter($stream)
$writer.write($binary)
$writer.Close()
# Write to the new CSV file to log the cust number and associated file
$NewLine=$CsvItem.'Customer Number'+","+$newFileName
$NewLine | Out-File-Append-FilePath$csvMappingFile-Encoding Unicode
Write-Host"$newFileName exported"
} catch {
Write-Host$_.Exception.Message
}
}
}
}
# Author: King Elder
# PLAN:
# - Iterates through CSV having customer numbers (NetSuite ID's)
# - Query SharePoint Library and filter on NetSuite ID
# - Loop through files to find Contracts or Addendums
# - Rename file if Customer Number not already in name
# - Export file to file system and write data to CSV file
####################### Variables ########################
$ExportedFileDestination="C:\\STAGING\\20140212ExportedFiles\\DocumentLibrary"
$csvMappingFile="C:\\STAGING\\20140212ExportedFiles\\SharePointExport.csv"
$SPWebUrl="http://SharePointSite"# <Url of the specific site>
$SPLibraryUrl="http://SharePointSite/DocumentLibrary/Documents/"
$CsvCustomerDataPathToFile="C:\\STAGING\\20140212ExportedFiles\\CustomerAccountNumbers.csv"
###########################################################
# Create mapping CSV with headers if doesn't exists
if (!(Test-Path-path$csvMappingFile))
{
"CustomerNumber, FileName">> $csvMappingFile
}
$web=Get-SPWeb-Identity$SPWebUrl
$list=$web.GetList($SPLibraryUrl)
$mpiData=Import-Csv-path$CsvCustomerDataPathToFile
Foreach ($CsvItemin$mpiData) {
$currentCustNumber=$CsvItem.'Customer Number'
# Filter the SharePoint list on the customer number (takes time since 60,361 items)
$FilteredListItems=$list.Items | ?{$_["CustNum"] -eq$currentCustNumber}
foreach($fileItemin$FilteredListItems)
{
if($fileItem.Name-Like"*addendum*"-or$fileItem.Name-Like"*contract*")
{
# Add Customer number if not in file name
if($fileItem.Name-Like"*$currentCustNumber*")
{
$newFileName=$fileItem.Name
}
else
{
$newFileName=$currentCustNumber+"."+$fileItem.Name
}
$newFilePath=$ExportedFileDestination+"\\"+$newFileName
$tempSourceFile="Documents/"+$fileItem.Name
# Download file
try {
$binary=$web.GetFile($tempSourceFile).OpenBinary()
$stream=New-ObjectSystem.IO.FileStream($newFilePath), Create
$writer=New-ObjectSystem.IO.BinaryWriter($stream)
$writer.write($binary)
$writer.Close()
# Write to the new CSV file to log the cust number and associated file
$NewLine=$CsvItem.'Customer Number'+","+$newFileName
$NewLine | Out-File-Append-FilePath$csvMappingFile-Encoding Unicode
Write-Host"$newFileName exported"
} catch {
Write-Host$_.Exception.Message
}
}
}
}