[新手上路]批处理新手入门导读[视频教程]批处理基础视频教程[视频教程]VBS基础视频教程[批处理精品]批处理版照片整理器
[批处理精品]纯批处理备份&还原驱动[批处理精品]CMD命令50条不能说的秘密[在线下载]第三方命令行工具[在线帮助]VBScript / JScript 在线参考
返回列表 发帖

[文本处理] 批处理去除文本的重复行并保留空行

本帖最后由 pcl_test 于 2018-1-6 01:31 编辑

网上的去重太多了,但是都把空行没了,现在需要保留空行!

回复 8# pcl_test


    牛逼 了。。。。

TOP

win7及以上系统,保存为bat文件运行
  1. <# :
  2. @echo off
  3. rem writen by pcl_test
  4. rem bat/ps1通用
  5. rem 测试前先备份源文件或文件夹
  6. type "%~f0"|powershell -noprofile -sta "-"
  7. >nul ping /n 6 0&exit
  8. #>
  9. $global:files=@();
  10. $ListFile={
  11.     $global:files=@();
  12.     $button2.Enabled = $false;
  13.     $button2.Text = '获取文件中';
  14.     if($radioButton1.Checked){
  15.         $global:files=FileDialog;
  16.     }else{
  17.         $global:files=FolderDialog;
  18.     }
  19.     $textBox2.Text=$files.Count;
  20.     if($files.Count){
  21.         $textBox3.Text=$files -join "`r`n";
  22.         $textBox3.Enabled = $true;
  23.     }else{
  24.         $textBox3.Text='测试前先备份源文件或文件夹';
  25.         $textBox3.Enabled = $false;
  26.     }
  27.     $button2.Text = '选择文件或文件夹';
  28.     $button2.Enabled = $true;
  29. }
  30. $EditFile={
  31.     switch($comboBox1.Text)
  32.     {
  33.         'UTF-8'{$outenc=[Text.Encoding]::UTF8;}
  34.         'Unicode(LittleEndian)'{$outenc=[Text.Encoding]::Unicode;}
  35.         'Unicode(BigEndian)'{$outenc=[Text.Encoding]::BigEndianUnicode;}
  36.         Default{$outenc=[Text.Encoding]::Default;}
  37.     }
  38.     if($files.Count){
  39.         $button1.Enabled = $false;
  40.         $button2.Enabled = $false;
  41.         $button1.Text = '处理中';
  42.         for($i=0;$i -lt $files.Count;$i++)
  43.         {
  44.             $tmp=ClearExtra $files[$i];
  45.             OutFile $files[$i] $tmp $outenc;
  46.             if([int]$textBox2.Text -gt 0){$textBox2.Text=[int]$textBox2.Text-1;}
  47.             [System.Windows.Forms.Application]::DoEvents();
  48.             write-host $files[$i];
  49.         }
  50.         $button1.Text = '执行';
  51.         $button1.Enabled = $true;
  52.         $button2.Enabled = $true;
  53.         [void][System.Windows.Forms.MessageBox]::Show('Done','提示');
  54.     }
  55. }
  56. function CheckExt
  57. {
  58.     $ext=@();
  59.     if(($textBox1.Text.Trim() -eq '') -or ($textBox1.Text.Trim() -notmatch '^\*\.[a-z\d]+(;\*\.[a-z\d]+)*$')){
  60.         [void][System.Windows.Forms.MessageBox]::Show('指定扩展名不能为空或非法');
  61.     }else{
  62.         $ext=($textBox1.Text -replace '\s*','').split(';',[StringSplitOptions]::RemoveEmptyEntries);
  63.     }
  64.     return ,$ext;
  65. }
  66. function FolderDialog
  67. {
  68.     $filelist=@();
  69.     $ext=CheckExt;
  70.     if($ext.Count){
  71.         $fbd = New-Object System.Windows.Forms.FolderBrowserDialog;
  72.         $fbd.RootFolder = 'MyComputer';
  73.         $fbd.ShowNewFolderButton = $false;
  74.         $Show = $fbd.ShowDialog();
  75.         If ($Show -eq 'OK'){
  76.             $fd=$fbd.SelectedPath;
  77.             dir $fd -r|?{($_ -is [System.IO.FileInfo]) -and ($ext -contains ('*'+$_.Extension))}|%{
  78.                 $filelist+=(,$_.FullName);
  79.             }
  80.         }
  81.     }
  82.     return ,$filelist;
  83. }
  84. function FileDialog
  85. {
  86.     $filelist=@();
  87.     $ext=CheckExt;
  88.     if($ext.Count){
  89.         $ofd = New-Object Windows.Forms.OpenFileDialog;
  90.         $ofd.InitialDirectory = pwd;
  91.         $ofd.Filter = '文本文件('+($ext -join ',')+')|'+$textBox1.Text;
  92.         $ofd.ShowHelp = $true;
  93.         $ofd.Multiselect = $true;
  94.         $Show=$ofd.ShowDialog();
  95.         if ($Show -eq 'OK'){
  96.             if ($ofd.Multiselect){$filelist=$ofd.FileNames;}else{$filelist=$ofd.FileName;}
  97.         }
  98.     }
  99.     return ,$filelist;
  100. }
  101. function ClearExtra($f)
  102. {
  103.     $dict=New-Object 'System.Collections.Generic.Dictionary[[string],[int]]';
  104.     $texttmp='';
  105.     $text=gc -LiteralPath $f;
  106.     if($radioButton3.Checked)
  107.     {
  108.         for($j=0;$j -lt $text.Count;$j++)
  109.         {
  110.             if($radioButton8.Checked)
  111.             {
  112.                 $line=$text[$j].Trim().ToLower();
  113.             }else{
  114.                 $line=$text[$j].Trim();
  115.             }
  116.             if($line -ne '')
  117.             {
  118.                 if(-not $dict.ContainsKey($line)){
  119.                     $texttmp+=$text[$j]+"`r`n";
  120.                     $dict.Add($line, 1);
  121.                 }
  122.             }else{
  123.                 if($radioButton5.Checked){$texttmp+=$text[$j]+"`r`n";}
  124.             }
  125.         }
  126.     }else{
  127.         for($j=0;$j -lt $text.Count;$j++)
  128.         {
  129.             if($radioButton8.Checked)
  130.             {
  131.                 $line=$text[$j].Trim().ToLower();
  132.             }else{
  133.                 $line=$text[$j].Trim();
  134.             }
  135.             if($line -ne '')
  136.             {
  137.                 if(-not $dict.ContainsKey($line)){
  138.                     $dict.Add($line, 1);
  139.                 }else{
  140.                     $dict[$line]++;
  141.                 }
  142.             }
  143.         }
  144.         for($j=0;$j -lt $text.Count;$j++)
  145.         {
  146.             if($radioButton8.Checked)
  147.             {
  148.                 $line=$text[$j].Trim().ToLower();
  149.             }else{
  150.                 $line=$text[$j].Trim();
  151.             }
  152.             if($line -ne '')
  153.             {
  154.                 if($dict[$line] -eq 1){$texttmp+=$text[$j]+"`r`n";}
  155.             }else{
  156.                 if($radioButton5.Checked){$texttmp+=$text[$j]+"`r`n";}
  157.             }
  158.         }
  159.     }
  160.     $dict=$null;
  161.     return $texttmp;
  162. }
  163. function OutFile($f, $txt, $enc)
  164. {
  165.     if($radioButton9.Checked)
  166.     {
  167.         $t=[int64]([DateTime]::UtcNow-(get-date '1970/1/1 0:0:0')).TotalMilliseconds;
  168.         $fpath=Split-Path $f;
  169.         $fname=Split-Path $f -Leaf;
  170.         $newname='$backup'+$t.ToString()+'_'+$fname;
  171.         cp $f ($fpath+'\'+$newname);
  172.     }
  173.     [IO.File]::WriteAllText($f, $txt, $enc);
  174. }
  175. Add-Type -Assembly 'System.Windows.Forms';
  176. $Form = New-Object System.Windows.Forms.Form;
  177. $Form.Text = '文本去除重复行';
  178. $Form.TopMost = $true;
  179. $Form.FormBorderStyle = 'FixedSingle';
  180. $Form.MaximizeBox = $false;
  181. $Form.Size = New-Object System.Drawing.Size(658,320);
  182. $radioButton1 = New-Object System.Windows.Forms.RadioButton;
  183. $radioButton1.AutoSize = $true;
  184. $radioButton1.AutoSize = $true;
  185. $radioButton1.Checked = $true;
  186. $radioButton1.Location = New-Object System.Drawing.Point(10, 22);
  187. $radioButton1.Name = 'radioButton1';
  188. $radioButton1.Size = New-Object System.Drawing.Size(95, 16);
  189. $radioButton1.Text = '指定的文本文件';
  190. $radioButton1.Add_Click({$textBox3.Text='测试前先备份源文件或文件夹'});
  191. $radioButton2 = New-Object System.Windows.Forms.RadioButton;
  192. $radioButton2.AutoSize = $true;
  193. $radioButton2.AutoSize = $true;
  194. $radioButton2.Location = New-Object System.Drawing.Point(120, 22);
  195. $radioButton2.Name = 'radioButton2';
  196. $radioButton2.Size = New-Object System.Drawing.Size(155, 16);
  197. $radioButton2.Text = '指定文件夹里的文本文件';
  198. $radioButton2.Add_Click({$textBox3.Text='注意:指定文件夹里的文本文件包括子文件夹里的'});
  199. $groupBox1 = New-Object System.Windows.Forms.GroupBox;
  200. $groupBox1.Location = New-Object System.Drawing.Point(12, 8);
  201. $groupBox1.Name = 'groupBox1';
  202. $groupBox1.Size = New-Object System.Drawing.Size(281, 50);
  203. $groupBox1.TabStop = $false;
  204. $groupBox1.Text = '选择处理对象';
  205. $groupBox1.Controls.Add($radioButton1);
  206. $groupBox1.Controls.Add($radioButton2);
  207. $Form.Controls.Add($groupBox1)
  208. $label1 = New-Object System.Windows.Forms.Label;
  209. $label1.AutoSize = $true;
  210. $label1.Location = New-Object System.Drawing.Point(8, 25);
  211. $label1.Name = 'label1';
  212. $label1.Size = New-Object System.Drawing.Size(35, 12);
  213. $label1.Text = '类型:';
  214. $textBox1 = New-Object System.Windows.Forms.TextBox;
  215. $textBox1.Location = New-Object System.Drawing.Point(43, 21);
  216. $textBox1.Name = 'textBox1';
  217. $textBox1.Size = New-Object System.Drawing.Size(230, 21);
  218. $textBox1.Text = '*.txt;*.log';
  219. $groupBox2 = New-Object System.Windows.Forms.GroupBox;
  220. $groupBox2.Location = New-Object System.Drawing.Point(12, 68);
  221. $groupBox2.Name = 'groupBox2';
  222. $groupBox2.Size = New-Object System.Drawing.Size(281, 50);
  223. $groupBox2.Text = '指定文件扩展名,多个中间用英文分号;分隔';
  224. $groupBox2.Controls.Add($label1);
  225. $groupBox2.Controls.Add($textBox1);
  226. $Form.Controls.Add($groupBox2)
  227. $radioButton3 = New-Object System.Windows.Forms.RadioButton;
  228. $radioButton3.AutoSize = $true;
  229. $radioButton3.Checked = $true;
  230. $radioButton3.Location = New-Object System.Drawing.Point(12, 20);
  231. $radioButton3.Name = 'radioButton3';
  232. $radioButton3.Size = New-Object System.Drawing.Size(119, 16);
  233. $radioButton3.Text = '重复行只保留一行';
  234. $radioButton4 = New-Object System.Windows.Forms.RadioButton;
  235. $radioButton4.AutoSize = $true;
  236. $radioButton4.Location = New-Object System.Drawing.Point(151, 20);
  237. $radioButton4.Name = 'radioButton4';
  238. $radioButton4.Size = New-Object System.Drawing.Size(119, 16);
  239. $radioButton4.Text = '重复行全清不保留';
  240. $groupBox3 = New-Object System.Windows.Forms.GroupBox;
  241. $groupBox3.Location = New-Object System.Drawing.Point(12, 129);
  242. $groupBox3.Name = 'groupBox3';
  243. $groupBox3.Size = New-Object System.Drawing.Size(281, 50);
  244. $groupBox3.Text = '选择处理方式';
  245. $groupBox3.Controls.Add($radioButton3);
  246. $groupBox3.Controls.Add($radioButton4);
  247. $Form.Controls.Add($groupBox3)
  248. $radioButton5 = New-Object System.Windows.Forms.RadioButton;
  249. $radioButton5.AutoSize = $true;
  250. $radioButton5.Checked = $true;
  251. $radioButton5.Location = New-Object System.Drawing.Point(6, 21);
  252. $radioButton5.Name = 'radioButton5';
  253. $radioButton5.Size = New-Object System.Drawing.Size(35, 16);
  254. $radioButton5.TabStop = $true;
  255. $radioButton5.Text = '是';
  256. $radioButton6 = New-Object System.Windows.Forms.RadioButton;
  257. $radioButton6.AutoSize = $true;
  258. $radioButton6.Location = New-Object System.Drawing.Point(48, 21);
  259. $radioButton6.Name = 'radioButton6';
  260. $radioButton6.Size = New-Object System.Drawing.Size(35, 16);
  261. $radioButton6.Text = '否';
  262. $groupBox4 = New-Object System.Windows.Forms.GroupBox;
  263. $groupBox4.Location = New-Object System.Drawing.Point(12, 192);
  264. $groupBox4.Name = 'groupBox4';
  265. $groupBox4.Size = New-Object System.Drawing.Size(89, 50);
  266. $groupBox4.Text = '保留空行';
  267. $groupBox4.Controls.Add($radioButton5);
  268. $groupBox4.Controls.Add($radioButton6);
  269. $Form.Controls.Add($groupBox4);
  270. $radioButton7 = New-Object System.Windows.Forms.RadioButton;
  271. $radioButton7.AutoSize = $true;
  272. $radioButton7.Location = New-Object System.Drawing.Point(6, 21);
  273. $radioButton7.Name = "radioButton7";
  274. $radioButton7.Size = New-Object System.Drawing.Size(35, 16);
  275. $radioButton7.Text = "是";
  276. $radioButton8 = New-Object System.Windows.Forms.RadioButton;
  277. $radioButton8.AutoSize = $true;
  278. $radioButton8.Checked = $true;
  279. $radioButton8.Location = New-Object System.Drawing.Point(51, 21);
  280. $radioButton8.Name = "radioButton8";
  281. $radioButton8.Size = New-Object System.Drawing.Size(35, 16);
  282. $radioButton8.Text = "否";
  283. $groupBox5 = New-Object System.Windows.Forms.GroupBox;
  284. $groupBox5.Location = New-Object System.Drawing.Point(110, 192);
  285. $groupBox5.Name = "groupBox5";
  286. $groupBox5.Size = New-Object System.Drawing.Size(89, 50);
  287. $groupBox5.Text = "区分大小写";
  288. $groupBox5.Controls.Add($radioButton7);
  289. $groupBox5.Controls.Add($radioButton8);
  290. $Form.Controls.Add($groupBox5);
  291. $radioButton9 = New-Object System.Windows.Forms.RadioButton;
  292. $radioButton9.AutoSize = $true;
  293. $radioButton9.Checked = $true;
  294. $radioButton9.Location = New-Object System.Drawing.Point(5, 21);
  295. $radioButton9.Name = 'radioButton9';
  296. $radioButton9.Size = New-Object System.Drawing.Size(35, 16);
  297. $radioButton9.Text = '是';
  298. $radioButton10 = New-Object System.Windows.Forms.RadioButton;
  299. $radioButton10.AutoSize = $true;
  300. $radioButton10.Location = New-Object System.Drawing.Point(47, 21);
  301. $radioButton10.Name = 'radioButton10';
  302. $radioButton10.Size = New-Object System.Drawing.Size(35, 16);
  303. $radioButton10.Text = '否';
  304. $groupBox6 = New-Object System.Windows.Forms.GroupBox;
  305. $groupBox6.Location = New-Object System.Drawing.Point(208, 192);
  306. $groupBox6.Name = 'groupBox6';
  307. $groupBox6.Size = New-Object System.Drawing.Size(85, 50);
  308. $groupBox6.Text = '备份源文件';
  309. $groupBox6.Controls.Add($radioButton9);
  310. $groupBox6.Controls.Add($radioButton10);
  311. $Form.Controls.Add($groupBox6);
  312. $label2 = New-Object System.Windows.Forms.Label;
  313. $label2.AutoSize = $true;
  314. $label2.Location = New-Object System.Drawing.Point(10, 258);
  315. $label2.Name = 'label2';
  316. $label2.Size = New-Object System.Drawing.Size(61, 12);
  317. $label2.Text = '输出编码:';
  318. $Form.Controls.Add($label2);
  319. $comboBox1 = New-Object System.Windows.Forms.ComboBox;
  320. $comboBox1.FormattingEnabled = $true;
  321. $comboBox1.Items.AddRange(@('Default(ANSI)','UTF-8','Unicode(LittleEndian)','Unicode(BigEndian)'));
  322. $comboBox1.Location = New-Object System.Drawing.Point(71, 255);
  323. $comboBox1.Name = 'comboBox1';
  324. $comboBox1.Size = New-Object System.Drawing.Size(150, 20);
  325. $comboBox1.Text = 'Default(ANSI)';
  326. $Form.Controls.Add($comboBox1);
  327. $button1 = New-Object System.Windows.Forms.Button;
  328. $button1.Location = New-Object System.Drawing.Point(226, 252);
  329. $button1.Name = 'button1';
  330. $button1.Size = New-Object System.Drawing.Size(67, 25);
  331. $button1.Text = '执行';
  332. $button1.Add_Click($EditFile);
  333. $Form.Controls.Add($button1);
  334. $button2 = New-Object System.Windows.Forms.Button;
  335. $button2.Location = New-Object System.Drawing.Point(300, 8);
  336. $button2.Name = 'button2';
  337. $button2.Size = New-Object System.Drawing.Size(182, 28);
  338. $button2.Text = '选择文件或文件夹';
  339. $button2.Add_Click($ListFile);
  340. $Form.Controls.Add($button2);
  341. $label3 = New-Object System.Windows.Forms.Label;
  342. $label3.AutoSize = $true;
  343. $label3.Location = New-Object System.Drawing.Point(532, 17);
  344. $label3.Name = 'label3';
  345. $label3.Size = New-Object System.Drawing.Size(59, 12);
  346. $label3.Text = '文件数量:';
  347. $Form.Controls.Add($label3);
  348. $textBox2 = New-Object System.Windows.Forms.TextBox;
  349. $textBox2.Location = New-Object System.Drawing.Point(594, 13);
  350. $textBox2.Name = 'textBox2';
  351. $textBox2.Size = New-Object System.Drawing.Size(46, 21);
  352. $textBox2.Text = '0';
  353. $Form.Controls.Add($textBox2);
  354.             
  355. $textBox3 = New-Object System.Windows.Forms.TextBox;
  356. $textBox3.Text='测试前先备份源文件或文件夹';
  357. $textBox3.Enabled = $false;
  358. $textBox3.Location = New-Object System.Drawing.Point(300, 40);
  359. $textBox3.Multiline = $true;
  360. $textBox3.Name = 'textBox3';
  361. $textBox3.ScrollBars = 'Both';
  362. $textBox3.Size = New-Object System.Drawing.Size(340, 234);
  363. $textBox3.WordWrap = $false;
  364. $Form.Controls.Add($textBox3);
  365. [void]$Form.ShowDialog();
  366. $Form.Dispose();
复制代码
2

评分人数

    • yhcfsr: 我竟然不知道原来PS还可以这样玩啊,够我研 ...技术 + 1
    • codegay: 1技术 + 1

TOP

回复 6# yhcfsr


    不能处理这些的文件名的文件还是文件了的这些内容?

TOP

本帖最后由 yhcfsr 于 2018-1-5 22:17 编辑

回复 5# ly88888
  1. @echo off&setlocal enableDelayedExpansion
  2. set "ScDir=d:\temp" rem 设置源目录
  3. set "OtFil=d:\temp.txt" rem 设置输出文件
  4. for /f "delims=" %%a in ('findstr /n ".*" "%ScDir%\test.txt"') do (
  5. for /f "tokens=1,2* delims=:" %%b in ("%%a") do (
  6. set "str=%%c"
  7. if "!str!" neq "" (if not defined [!str!] (
  8. set [!str!]=!str!
  9. call echo %%[!str!]%%
  10. )
  11. ) else (echo.)
  12. ))>>%OtFil%
  13. pause
复制代码
上面代码的缺陷是不能处理!&#<>等特殊英文字符
想要处理特殊字符,用powershell脚本或VBS做吧

TOP

回复 4# yhcfsr


    不相邻行不存在重复 当然有重复的话也一起去掉

TOP

回复 3# ly88888


    你是想删除相邻两行重复的一行吧?如果不相邻的行有重复行,需要删除吗?

TOP

名言30句
名言30句
真理惟一可靠的标
准就是永远自相符合

歌词30句
歌词30句
土地是以它的肥沃和收获而被估价的
;才能也是土地,不过它生产的不是粮食,
而是真理。如果只能滋生瞑想和幻想的话,



诗词60句
诗词60句
时间是一切财富中
最宝贵的财富

——————————————————————————

处理后的结果

——————————————————————————

名言30句
真理惟一可靠的标
准就是永远自相符合

歌词30句
土地是以它的肥沃和收获而被估价的
;才能也是土地,不过它生产的不是粮食,
而是真理。如果只能滋生瞑想和幻想的话,




诗词60句
时间是一切财富中
最宝贵的财富

——————————————————————————

而不是

——————————————————————————


名言30句
真理惟一可靠的标
准就是永远自相符合
歌词30句
土地是以它的肥沃和收获而被估价的
;才能也是土地,不过它生产的不是粮食,
而是真理。如果只能滋生瞑想和幻想的话,
诗词60句
时间是一切财富中
最宝贵的财富
————————————

TOP

保留空行不难,但你要把要实现的要求写具体点。

TOP

返回列表