@echo off
setlocal ENABLEDELAYEDEXPANSION

set sour=aaa
set dest=bbb
del %dest%\*.txt

set exclude=20210226

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%sour%\*.txt"`) do (
    sed -r -n -e "/\w+\s+\w+\s+\w+/p" -e "/%exclude%/d" "%%~fa">> "%dest%\%%~na.txt"
)

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%dest%\*.txt"`) do (
    set _file_=%%~fa
    for /f "usebackq tokens=* delims=" %%a in ("%_file_%") do (
        set __%%a=%%a
    )
    
    (
        for /f "usebackq tokens=2 delims==" %%a in (`set __`) do (
            echo %%a
        )
    ) > "%_file_%"
    rem notepad"%_file_%"
)
复制代码

@echo off
setlocal ENABLEDELAYEDEXPANSION

set sour=C:\Users\admin\Desktop\New folder (2)\aaa
set dest=C:\Users\admin\Desktop\New folder (2)\bbb
set exclude=20210226

del "%dest%\*.txt"

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%sour%\*.txt"`) do (
    sed -r -n -e "/\w+\s+\w+\s+\w+/p" -e "/%exclude%/d" "%%~fa">> "%dest%\%%~na.txt"
)

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%dest%\*.txt"`) do (
    for /f "usebackq tokens=* delims=" %%a in ("%%~fa") do (
        set __%%a=%%a
    )
    
    (
        for /f "usebackq tokens=2 delims==" %%a in (`set __`) do (
            echo %%a
        )
    ) > "%%~fa"
    notepad.exe "%%~fa"
)
复制代码

$sour = "aaa"
$dest = "bbb"
$exclude = "20210227"

Remove-Item $dest\*.txt

Get-ChildItem -path $sour *.txt -Recurse | foreach-object {
    ( get-content $_.fullname ) -match "\w+\s+\w+\s+[-]?\w+" -notmatch $exclude | out-file -append $dest\$_
}

Get-ChildItem -path $dest *.txt | foreach-object {
    $a = get-content $_.fullname | sort-object -unique
    $a | out-file $dest\$_
    notepad.exe $dest\$_
}
复制代码

├───aaa
│   ├───01
│   └───02
└───bbb
复制代码

@echo off
setlocal ENABLEDELAYEDEXPANSION

set sour=C:\Users\admin\Desktop\New folder (2)\aaa
set dest=C:\Users\admin\Desktop\New folder (2)\bbb
set exclude=20210226

del "%dest%\*.txt"

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%sour%\*.txt"`) do (
    sed -r -n -e "/\w+\s+\w+\s+\w+/p" -e "/%exclude%/d" "%%~fa">> "%dest%\%%~na.txt"
)

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%dest%\*.txt"`) do (
    for /f "usebackq tokens=2 delims==" %%a in (`set __ 2^>nul`) do (
        set %%a=
    )
    for /f "usebackq tokens=* delims=" %%a in ("%%~fa") do (
        set __%%a=1
    )
    
    (
        for /f "usebackq tokens=2 delims==" %%a in (`set __`) do (
            echo %%a
        )
    ) > "%%~fa"
    notepad.exe "%%~fa"
)
复制代码

$sour = "D:\share\tech\New folder (2)\aaa"
$dest = "D:\share\tech\New folder (2)\ccc"
$exclude = "20210227"

Remove-Item $dest\*.txt

get-date

Get-ChildItem -path $sour *.txt -Recurse | foreach-object {
    $_.fullname
    ( get-content $_.fullname ) -match "\w+\s+\w+\s+[-]?\w+" -notmatch $exclude -replace "SZ","1" -replace "SH","0" -replace "\s+","|" | out-file -append $dest\$_
}

get-date

Get-ChildItem -path $dest *.txt | foreach-object {
    $_.fullname
    $a = get-content $_.fullname | sort-object -unique
    $a | out-file $dest\$_
}

get-date
复制代码

├───aaa    <--- 数据源
     ├───01
     └───02
复制代码

set Path=C:\msys64\usr\bin;%Path%
复制代码

@echo off
setlocal ENABLEDELAYEDEXPANSION

set path=%pathgnu%;%path%
echo %path%

set sour=新建文件夹
set dest=bbb1
set exclude=QQ

del /q "%dest%\*.*"

echo %~nx0 %time% 1 >>time.txt

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%sour%\*.txt"`) do (
    echo "%%~fa"
    type "%%~fa" | sed -r -e "/\S+\s+\S+\s+\S+/^!d" -e "/%exclude%/d" -e "s/SZ/1/" -e "s/SH/0/" -e "s/\s+/^|/g"  >> "%dest%\%%~na.tmp"
)

echo,
echo %~nx0 %time% 2 >>time.txt
echo,

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%dest%\*.tmp"`) do (
    echo "%%~fa"
    type "%%~fa" | sort.exe -u >>"%%~dpna.txt"
)

echo %~nx0 %time% 3 >>time.txt
echo,
del %dest%\*.tmp
复制代码

@echo off
setlocal ENABLEDELAYEDEXPANSION

set path=%pathgnu%;%path%
echo %path%

set sour=新建文件夹
set dest=bbb2
set exclude=QQ

del /q "%dest%\*.*"

echo %~nx0 %time% 1 >>time.txt

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%sour%\*.txt"`) do (
    echo "%%~fa"
    type "%%~fa" | sed -r -e "/\S+\s+\S+\s+\S+/^!d" -e "/%exclude%/d" -e "s/SZ/1/" -e "s/SH/0/" -e "s/\s+/^|/g"  >> "%dest%\%%~na.tmp"
)

echo,
echo %~nx0 %time% 2 >>time.txt
echo,

for /f "usebackq tokens=* delims=" %%a in (`dir /s /b "%dest%\*.tmp"`) do (
    echo "%%~fa"
    type "%%~fa" | awk " { arr[$0]++ ; if ( arr[$0] == 1 ) { print $0 } } "  >>"%%~dpna.txt"
)

echo %~nx0 %time% 3 >>time.txt
echo,
del %dest%\*.tmp
复制代码

Get-ChildItem -path $sour *.txt -Recurse | foreach-object {
    $_.fullname
    $b=( [IO.File]::ReadAllLines($_.fullname) ) -match "\w+\s+\w+\s+[-]?\w+" -notmatch $exclude -replace "SZ","1" -replace "SH","0" -replace "\s+","|"
    $a=$dest,$_.name -join '\'
    [IO.File]::WriteAllLines($a,$b)
}
复制代码

$sour = "D:\share\tech\New folder\新建文件夹"
$dest = "D:\share\tech\New folder\ccc2"
$exclude = "QQ"

Remove-Item $dest\*.*

( $MyInvocation.MyCommand.Name + "  1  " + (get-date -Format "HH:mm:ss.ff").tostring() ) | out-file -Encoding ascii -append time.txt

Get-ChildItem -path $sour *.txt -Recurse | foreach-object {
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".tmp")
    $a=( [IO.File]::ReadAllLines($_.fullname) )
    $a = $a -match "\S+\s+\S+\s+\S+" -notmatch $exclude -replace "SZ","1" -replace "SH","0" -replace "\s+","|"
    [IO.File]::AppendAllLines([string]$filename , [string[]]$a)
}

( $MyInvocation.MyCommand.Name + "  2  " + (get-date -Format "HH:mm:ss.ff").tostring() ) | out-file -Encoding ascii -append time.txt

Get-ChildItem -path $dest *.tmp | foreach-object {
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".txt")
    $ht = @{}
    $a = ( [IO.File]::ReadAllLines($_.fullname) )
    $a = $a | foreach-object {
        if ( -not ( $ht.ContainsKey($_) ) )
        {
            $_
            $ht.add($_,"1")
        }
    }
    $a | out-file -Encoding utf8 $filename
}

( $MyInvocation.MyCommand.Name + "  3  " + (get-date -Format "HH:mm:ss.ff").tostring() ) | out-file -Encoding ascii -append time.txt
复制代码

Get-ChildItem -path $dest *.tmp | foreach-object {
    $reader = New-Object -TypeName System.IO.StreamReader -ArgumentList $_.fullname
    $aa=New-Object System.Collections.Generic.HashSet[string]
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".txt")
    while ( $read = $reader.ReadLine() ) {
        out-null -InputObject $aa.add($read)
    }
    [IO.File]::WriteAllLines($filename,$aa)
}
复制代码

#&cls&cd /d "%~dp0" & @powershell -c "Get-Content '%~0' | Select-Object -Skip 1 | Out-String | Invoke-Expression" & pause&exit
cls
$t1 = Get-Date
$src_dir = '新建文件夹'
$dst_dir = 'out'

[void][System.IO.Directory]::CreateDirectory($dst_dir)

#线程函数 处理数据
$HandleGroupJob = {
    #线程参数
    param($dst_dir,$groupInfo)
    Write-Host $groupInfo.Name
    #汇总去重并筛选
	$set = New-Object 'System.Collections.Generic.HashSet[string]'
    & { $groupInfo.Group | foreach { [IO.File]::ReadAllLines($_.FullName)} } | foreach {
		if($_ -match '\S+\s+\d+\s+-?\d{2,}'){
			[void]$set.Add(($_ -replace 'SH','1|' -replace 'SZ','0|' -replace '\s+','|'))
		}
	}
    #输出
    Out-File -InputObject $set -FilePath ('{0}\{1}' -f $dst_dir,$groupInfo.Name) 
	$set = $null
    return ($groupInfo.Name + ' 已完成')
}

#多线程设置
$pool = [runspacefactory]::CreateRunspacePool(1,10) #最多10个线程并发
$pool.Open()
$threads = New-Object 'System.Collections.ArrayList'
$results = New-Object 'System.Collections.ArrayList'

'开始创建线程...'
Get-ChildItem -Recurse -Path $src_dir -Filter '*.txt' | Group-Object {$_.Name} | foreach {
    $_.Name
    $thread = [powershell]::Create()
    $thread.RunspacePool = $pool
    [void]$thread.AddScript($HandleGroupJob)
    [void]$thread.AddArgument($dst_dir)
    [void]$thread.AddArgument($_)
    [void]$threads.Add($thread)
    [void]$results.Add($thread.BeginInvoke())
}
'-------------------------------'

'等待线程结束'
while($true){
    $all_done = $true
    for($i = 0; $i -lt $results.Count; $i++){
        if($results[$i] -ne $null){
            if($results[$i].IsCompleted){
                $threads[$i].EndInvoke($results[$i])
				$threads[$i].Dispose()
				$threads[$i] = $null
                $results[$i] = $null
				[System.GC]::Collect()
            } else {
                $all_done = $false
            }
        }
    }
    if($all_done){ break }
    Start-Sleep -Milliseconds 500
}

#关闭线程池
$pool.Close()
'-------------------'
'{0}  -> {1}' -f $t1,(Get-Date)
复制代码

Get-ChildItem -path $dest *.tmp | foreach-object {
    $reader = New-Object -TypeName System.IO.StreamReader -ArgumentList $_.fullname
    $aa=New-Object System.Collections.Generic.HashSet[string]
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".txt")
    while ( $read = $reader.ReadLine() ) {
        [void]$aa.add($read)
    }
    [IO.File]::WriteAllLines($filename,$aa)
}
复制代码

 $null = 
复制代码

Rem On Error Resume Next
Dim srcDir, dstDir

srcDir = "E:\Test\新建文件夹"           '源文件夹路径，需替换为实际路径
dstDir = "E:\Test\新建文件夹1"          '目标文件夹路径

Dim wsc
Set wsc = CreateObject("WScript.Shell")

If WSH.Arguments.Count = 0 Then
    wsc.Run "cscript " & chr(34) & WSH.ScriptFullName & chr(34) & " ARG", 0
    WScript.Quit
End If

Dim dic1, fso
Set dic1 = CreateObject("Scripting.Dictionary")
Set fso  = CreateObject("Scripting.FileSystemObject")

Dim reg
Set reg     = New RegExp
reg.Pattern = "[ \t]+"
reg.Global  = True

Dim objExec
If Not fso.FolderExists(srcDir) Then MsgBox "源文件夹不存在" : WSH.Quit
wsc.Run "cmd /c md " & chr(34) & dstDir & chr(34) & " 2>nul", 0              '创建目标文件夹
Set objExec = wsc.Exec( "cmd /c dir /b /a-d /s " & chr(34) & srcDir & "\*.txt" & chr(34) )  '遍历源文件夹

Rem 字典dic1，key=文件名，value=文件路径
Dim strFilePath, strFileName
Do Until objExec.StdOut.AtEndOfStream
    strFilePath = objExec.StdOut.ReadLine
    strFileName = fso.GetFile(strFilePath).Name
    strFileName = LCase(strFileName)
    If Not dic1.Exists(strFileName) Then
        dic1.Add strFileName, strFilePath
    Else
        dic1.Item(strFileName) = dic1.Item(strFileName) & vbLf & strFilePath
    End If
Loop

Rem 遍历字典dic1
Dim key
For Each key In dic1.Keys 
    GetFileContent key, Split( Dic1.Item(key), vbLf )
Next

Function GetFileContent(fileName, ByRef arrFilePath)
    Dim dic2, i, objFile, strLine
    Set dic2 = CreateObject("Scripting.Dictionary")

    For i = 0 To UBound(arrFilePath)
        Set objFile = fso.OpenTextFile(arrFilePath(i), 1)
        Do Until objFile.AtEndOfStream
            strLine = objFile.ReadLine
            strLine = reg.Replace(strLine, "|")                                       '空格替换为"|"
            If UBound(Split(strLine, "|")) >= 2 And InStr(strLine, "-QQ") <= 0 Then   '如果大于等于3列、不含 "-QQ"
                strLine = Replace(Replace(strLine,"SZ", "0|"),"SH", "1|")             '替换 "SZ" 和 "SH"
                If Not dic2.Exists(strLine) Then                                      '字典dic2，用于去重复
                    dic2.Add strLine, True
                End If
            End If 
        Loop
        objFile.Close
        Set objFile = Nothing
    Next

    fso.OpenTextFile(dstDir & "\" & fileName, 2, True).Write( Join(dic2.Keys, vbCrLf) )    '保存
    Set dic2 = Nothing
End Function

MsgBox "Done"
复制代码

$dt = get-Date;

$srcDir = 'E:\Test\新建文件夹';
$dstDir = 'E:\Test\新建文件夹1';

If (![IO.Directory]::Exists($dstDir)){ $null = md $dstDir }

$dic = New-Object 'System.Collections.Generic.Dictionary[string, [Collections.ArrayList]]';

forEach( $file In (dir $srcDir -Recurse -Filter '*.txt') ){
    $Name = $file.BaseName.ToUpper();
    $Path = $file.FullName;
    if( !$dic.ContainsKey($Name) ){
        $dic.Add($Name, @($Path));
    } else {
        [void]$dic[$Name].Add($Path);
    }
}

forEach( $key In $dic.Keys ){
    $Hash = @{};
    for( $i=0; $i -lt $dic[$key].Count; $i++ ){
        $arr = [IO.File]::ReadAllLines($dic[$key][$i], [Text.Encoding]::Default);
        $arr = $arr -match '^(?:(?!-QQ)\S)+\s+\S+\s+\S+';
        $arr = $arr -replace '\s+', '|' -replace 'SZ', '0|' -replace 'SH', '1|';
        $Count = $arr.Count;
        for( $j=0; $j -lt $Count; $j++ ){
            if( !$Hash.ContainsKey($arr[$j]) ){
                $Hash.Add($arr[$j], $True);
            }
        }
    }
    [IO.File]::WriteAllLines( $dstDir + '\' + $key + '.txt', $Hash.Keys );
}
((get-Date) - $dt).TotalSeconds;
复制代码

@echo off
setlocal ENABLEDELAYEDEXPANSION

set sour=新建文件夹
set dest=%~n0
set exclude=QQ

set "_cmd_=dir /s /b %sour%\*.txt | sed -r -e "s/^.+[\]//" | sort -u"
set _cmdsed_=-r -e "/%exclude%/d" -e "/^\S+\s+\S+\s+\S+$/^!d" -e "s/SZ/1/" -e "s/SH/0/" -e "s/\s+/^|/g"
set _cmdawk_=" { arr[$0]++ ; if ( arr[$0] == 1 ) { print $0 } } "
for /f "usebackq tokens=* delims=" %%a in (`cmd /c "%_cmd_%"`) do (
    echo %%a
    (
        for /f "usebackq tokens=* delims=" %%b in (`dir /s /b "%sour%\%%a"`) do (
            type "%%~fb"
        ) | sed -r -e $a\
    ) | sed !_cmdsed_! | awk !_cmdawk_! > "%dest%\%%~nxa"
)
复制代码

@echo off
setlocal ENABLEDELAYEDEXPANSION
chcp 936

set path=%pathgnu%;%path%

set sour=新建文件夹
set dest=%~n0
set exclude=QQ

if exist %dest% rd /s /q %dest%
if not exist %dest% mkdir %dest%

call :C_timer

set "_cmd_=dir /s /b %sour%\*.txt | sed -r -e "s/^.+[\]//" | sort -u"
set _cmdsed_=-r -e "/%exclude%/d" -e "/^\S+\s+\S+\s+\S+$/^!d" -e "s/SH/0/" -e "s/SZ/1/" -e "s/\s+/^|/g"
set _cmdawk_=" { arr[$0]++ ; if ( arr[$0] == 1 ) { print $0 } } "
for /f "usebackq tokens=* delims=" %%a in (`cmd /c "%_cmd_%"`) do (
    echo %%a
    (
        for /f "usebackq tokens=* delims=" %%b in (`dir /s /b "%sour%\%%a"`) do (
            type "%%~fb"
        ) | sed -r -e $a\
    ) | sed !_cmdsed_! | awk !_cmdawk_! > "%dest%\%%~nxa"
)

call :C_timer sum

exit/b

rem del %dest%\*.tmp


:C_timer
if not defined _ti_ set _ti_=-1
set/a _ti_+=1
set _tb_=%_te_%
set _te_=%time%
if %_ti_% EQU 0 (
    set _tb_=%_te_%
)
set/a _tdiff_=(9%_te_:~0,2%-9%_tb_:~0,2%)*360000+(9%_te_:~3,2%-9%_tb_:~3,2%)*6000+(9%_te_:~6,2%%_te_:~9,2%-9%_tb_:~6,2%%_tb_:~9,2%)
set/a _tdiff_=%_tdiff_:-=8640000-%
set/a _tdiffSum_+=%_tdiff_%
if "%_ti_%" == "0" (
    (echo,) >>time.txt
    (echo %~nx0  %_ti_%  %_te_%) >>time.txt
) else (
    (echo %~nx0  %_ti_%  %_te_%  %_tdiff_:~0,-2%.%_tdiff_:~-2%) >>time.txt
)
if %1.==sum. (
    (echo %~nx0                  %_tdiffSum_:~0,-2%.%_tdiffSum_:~-2%) >>time.txt
)
goto :eof
复制代码