@echo off&setlocal enabledelayedexpansion
echo 正在创建测试文件 y1.txt 请稍候。。。 行数为 1000005 行
(for /l %%a in (1 1 100) do echo !random!----!random!----!random!!random!----!random!)>y1.txt
(for /l %%a in (1 1 10) do type y1.txt)>y2.txt
(for /l %%a in (1 1 10) do type y2.txt)>y1.txt
(for /l %%a in (1 1 10) do type y1.txt)>y2.txt
(for /l %%a in (1 1 10) do type y2.txt)>y1.txt
(echo !random!----!random!----!random!!random!----!random!!random!
echo !random!----!random!----!random!!random!----!random!!random!
echo !random!----!random!----!random!!random!----!random!!random!
echo !random!----!random!----!random!!random!----!random!!random!
echo !random!----!random!----!random!!random!----!random!!random!)>>y1.txt
del /q y2.txt
复制代码

@echo off&setlocal enabledelayedexpansion
(for /f "delims=" %%a in (sour2.txt) do (
        set/a n+=1
        set num=000000!n!
        echo !num:~-7!-%%a
))>temp.txt
::加入序号保持原来的顺序

(for /f "tokens=1,2,3* delims=-" %%a in ('sort temp.txt /+8') do (
        if "%%b-%%c" neq "!var!" (echo %%a-%%b-%%c-%%d)
        set var=%%b-%%c
))>temp2.txt
::从第8位开始排序，即只按原来的数据排序，重复的丢弃

(for /f "tokens=1,2,3* delims=-" %%a in ('sort temp2.txt') do (
echo %%b----%%c----%%d
))>temp3.txt
::恢复原来的顺序，并丢弃加入的临时序号
复制代码

 
@echo off
(for /f "delims=" %%a in (sour.txt) do (
for /l %%b in (1,1,10000) do (echo %%a)
))>sour2.txt
复制代码

gawk "BEGIN {FS=\"-\"} !a[$1 $2]++" y1.TXT>y2.txt
复制代码

@echo off&setlocal enabledelayedexpansion
echo 正在创建测试文件 y1.txt 请稍候
for /l %%a in (1 1 1000) do (
(for /l %%a in (1 1 2001) do echo !random!!random!!random!----!random!!random!!random!----!random!----!random!!)>>y1.txt 
(for /l %%a in (1 1 1001) do echo %random%----%random%----!random!!random!!random!----!random!!random!!random!)>>y1.txt
)
复制代码

@echo off
for /f "delims=" %%i in (test.txt) do (
    if not defined %%i set %%i=s & echo %%i>>result.txt)
复制代码

Const adOpenStatic = 3
Const adLockOptimistic = 3
Const adCmdText = &H0001

Set objConnection = CreateObject("ADODB.Connection")
Set objRecordSet = CreateObject("ADODB.Recordset")

strPathToTextFile = "C:\Scripts\"
strFile = "Test.txt"

objConnection.Open "Provider=Microsoft.Jet.OLEDB.4.0;" & _
      "Data Source=" & strPathtoTextFile & ";" & _
          "Extended Properties=""text;HDR=NO;FMT=Delimited"""

objRecordSet.Open "Select DISTINCT * FROM " & strFile, _
    objConnection, adOpenStatic, adLockOptimistic, adCmdText

Do Until objRecordSet.EOF
    Wscript.Echo objRecordSet.Fields.Item(0).Value   
    objRecordSet.MoveNext
Loop
复制代码

@echo off&setlocal enabledelayedexpansion
(for /f "delims=" %%a in (y1.txt) do (
	set /a "n=(10000001+n)-(n/10000000)*10000000"
	echo !n:~-7!:%%a
))>1.tmp
sort /rec 8198 /+8 1.tmp>2.tmp
(for /f "tokens=1* delims=:" %%a in (2.tmp) do (
	if "%%b" neq "!l!" (
		echo %%a
		set l=%%b
	)
))>3.tmp
findstr /b /g:3.tmp 1.tmp>4.tmp
(for /f "tokens=1* delims=:" %%a in (4.tmp) do echo %%b)>y2.txt
del /f /q ?.tmp
(for /f "delims=" %%a in (y2.txt) do (
	set /p=%%a:<nul
	for /f "tokens=3" %%b in ('find /c "%%a" y1.txt') do echo 存在%%b行重复
))>check.txt
pause
复制代码