<# .SYNOPSIS Deduplicates files based on a common source directory. .DESCRIPTION Deletes duplicated files in dependent directories which are also contained in the specified common directory and generates a links.json file listing all of the deduplications. .PARAMETER commonDir Source directory to which links are created. .PARAMETER DependentDir Directory containing subdirectories of dependents to scan for matches. #> param([string]$CommonDir, [string]$DependentDir) $hashes = @{} foreach ($item in (Get-ChildItem -Path $CommonDir -File -Recurse)) { $hash = (Get-FileHash -Path $item.FullName) if(-not $hashes.ContainsKey($hash.Hash)) { $hashes.Add($hash.Hash, $hash.Path) } } foreach ($depDir in (Get-ChildItem -Path $DependentDir -Directory)) { $fileLinks = @{} foreach ($item in (Get-ChildItem -Path $depDir.FullName -File -Recurse)) { $hash = (Get-FileHash -Path $item.FullName) if ($hashes.ContainsKey($hash.Hash)) { $commonItemPath = $hashes[$hash.Hash].Substring($CommonDir.Length + 1) $depItemPath = $item.FullName.Substring($depDir.FullName.Length + 1); $fileLinks.Add($depItemPath, $commonItemPath) Remove-Item -Path $item.FullName } } Write-Output "Deduplicated $($fileLinks.Count) items in $($depDir.Name)" ConvertTo-Json $fileLinks | Out-File -FilePath (Join-Path -Path $depDir.FullName -ChildPath 'links.json') -Encoding utf8 } Remove-Variable -Name hashes