Mojo::UserAgent 批量获取 nes 游戏资源

523066680

获取midi（背景音乐）、nes、图片、作弊码、以及PDF（通常是手册）
运行环境：Strawberry Perl v5.26
路径设置：our $wdir = "F:/temp/nesgames";

1. =info
   
2.     Mojo::UserAgent 批量获取 nes 游戏资源
   
3.     523066680/vicyang
   
4.     2018-12
   
5. =cut
   
6. 
   
7. use File::Slurp;
   
8. use File::Path;
   
9. use File::Basename;
   
10. use Mojo::UserAgent;
    
11. use Mojo::DOM;
    
12. use Try::Tiny;
    
13. STDOUT->autoflush(1);
    
14. 
    
15. our $wdir = "F:/temp/nesgames";
    
16. our $main = "http://www.nesfiles.com";
    
17. our $games = "http://www.nesfiles.com/Games";
    
18. 
    
19. mkpath $wdir unless -e $wdir;
    
20. chdir $wdir;
    
21. 
    
22. our $ua = Mojo::UserAgent->new();
    
23. our @headers = ( "User-Agent" => "Firefox/63.0" );
    
24. 
    
25. get_games_list($games);
    
26. 
    
27. sub get_games_list
    
28. {
    
29.     our ($ua, $main, @heaee);
    
30.     my ($link) = @_;
    
31.     my $res = $ua->get( $link, \@headers )->res;
    
32.     my $dom = $res->dom;
    
33. 
    
34.     for my $e ( $dom->find(".nesfilesTable a")->each )
    
35.     {
    
36.         #printf "%s %s\n", $e->attr("href"), $e->text;
    
37.         get_files( $main .$e->attr("href"), $e->text );
    
38.     }
    
39. }
    
40. 
    
41. sub get_files
    
42. {
    
43.     our ($main, $ua, @headers);
    
44.     my ($link, $name) = @_;
    
45.     my $title = basename($link);
    
46.     my ($res, $dom);
    
47. 
    
48.     my $fname = "${title}.html";
    
49.     if ( -e $fname ) {
    
50.         my $html = read_file($fname);
    
51.         $dom = Mojo::DOM->new( $html );
    
52.     } else {
    
53.         $res = $ua->get( $link, \@headers )->res;
    
54.         $dom = $res->dom;
    
55.         write_file( $fname, {binmode=>":raw"}, $res->body );
    
56.     }
    
57. 
    
58.     # 获取资源明细，略过 Ebay 相关的条目
    
59.     mkdir $title unless -e $title;
    
60.     my ($head, $list, $res2);
    
61.     for my $section ($dom->find(".GameSection")->each)
    
62.     {
    
63.         $head = $section->at("header")->text;
    
64.         last if $head=~/Ebay$/i;
    
65. 
    
66.         # 秘籍/代码
    
67.         if ($head=~/Codes/i) {
    
68.             write_file( $title ."/Codes_Cheats.txt", $section->all_text );
    
69.             next;
    
70.         }
    
71. 
    
72.         if ($head=~/Screenshots/i) {
    
73.             # 如果是屏幕截图
    
74.             $list = $section->find("img")->map(attr=>"src");
    
75.         } else {
    
76.             # 其他情况获取 href
    
77.             $list = $section->find("a")->map(attr=>"href");
    
78.         }
    
79. 
    
80.         printf "%s\n", $head;
    
81.         for my $href ( $list->each )
    
82.         {
    
83.             printf "%s\n", $href;
    
84.             $fname = $title ."/". basename($href);
    
85.             
    
86.             next if -e $fname;         # 跳过已经存在的文件
    
87.             $res2 = try_to_get( "${main}$href" );
    
88.             next unless defined $res2; # 如果获取失败
    
89. 
    
90.             write_file( $fname, {binmode=>":raw"}, $res2->body);
    
91.         }
    
92.     }
    
93. }
    
94. 
    
95. sub try_to_get
    
96. {
    
97.     our ($ua, @headers);
    
98.     my ($link) = @_;
    
99.     my $res;
    
100.     my $times = 0;
     
101. 
     
102.     while (1)
     
103.     {
     
104.         try { $res = $ua->get( $link )->result; }
     
105.         catch { printf "Error %s, retry: %d\n", $_, $times; };
     
106.         $times++;
     
107.         last if (defined $res and $res->is_success);
     
108.         return undef if ( $times > 5 );
     
109.     }
     
110.     return $res;
     
111. }
     

复制代码