added sn_uploader

diff --git a/bin/3ch-idx.rb b/bin/3ch-idx.rb
new file mode 100755 (executable)
index 0000000..e0524b2
--- /dev/null
+++ b/bin/3ch-idx.rb
@@ -0,0 +1,32 @@
+#!/usr/bin/ruby
+# download from 3ch
+# http://www.esc-j.net/tech-arts/ta3dc/t1a931d9c1s9.html
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_fileno(text)
+  file_href = 'http://www.esc-j.net/tech-arts/ta3dc/upload.cgi?mode=dl&file='
+  file_re = Regexp.new(Regexp.escape(file_href) + '(\d+)')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "www.esc-j.net"
+uploader.root_path = "/tech-arts/ta3dc"
+uploader.base_html = "t1a931d9c1s9.html"
+uploader.kcode = 'U'
+uploader.authorization = "Basic " + ["tech:mybride"].pack('m').chomp
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/3ch'
+body = uploader.get_base
+each_fileno(body) do |fileno|
+  basename_without_extension = "TA3CH%04d" % fileno.to_i
+  filename_re = Regexp.new('\A' + Regexp.escape(basename_without_extension) + '\.')
+  exist = uploader.local_file_match?(filename_re)
+  puts [ fileno, exist ? 'exist' : 'download' ].join("\t")
+  uploader.download(fileno) unless exist
+end

diff --git a/bin/hnt-idx.rb b/bin/hnt-idx.rb
new file mode 100755 (executable)
index 0000000..caf089a
--- /dev/null
+++ b/bin/hnt-idx.rb
@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+# download from futabacustom save
+# http://www.nijibox5.com/futabacustom/save/
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_filename(text)
+  file_href = './src/'
+  file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "www.nijibox5.com"
+uploader.root_path = "/futabacustom/save"
+uploader.base_html = ""
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/hnt'
+body = uploader.get_base
+each_filename(body) do |filename|
+  exist = uploader.local_file_exist?(filename)
+  puts [ filename, exist ? 'exist' : 'download' ].join("\t")
+  uploader.download_file(filename) unless exist
+end

diff --git a/bin/mmd-idx.rb b/bin/mmd-idx.rb
new file mode 100755 (executable)
index 0000000..bf054d9
--- /dev/null
+++ b/bin/mmd-idx.rb
@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+# download from mikumikudance
+# http://bytatsu.net/uploader/mikumikudance/index.html
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_filename(text)
+  file_href = './src/'
+  file_re = Regexp.new(Regexp.escape(file_href) + '(.+)')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "bytatsu.net"
+uploader.root_path = "/uploader/mikumikudance"
+uploader.base_html = "index.html"
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/mmd'
+body = uploader.get_base
+each_filename(body) do |filename|
+  exist = uploader.local_file_exist?(filename)
+  puts [ filename, exist ? 'exist' : 'download' ].join("\t")
+  uploader.download_file_directly(filename) unless exist
+end

diff --git a/bin/mmdacc1-idx.rb b/bin/mmdacc1-idx.rb
new file mode 100755 (executable)
index 0000000..0fda001
--- /dev/null
+++ b/bin/mmdacc1-idx.rb
@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+# download from mikumikudance
+# http://www9.atpages.jp/~mmdaccessory/uploader/index.php
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_filename(text)
+  file_href = './log/'
+  file_re = Regexp.new(Regexp.escape(file_href) + '(.+)')
+  text.scan(/href='(.+?)'/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "www9.atpages.jp"
+uploader.root_path = "/~mmdaccessory/uploader"
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/mmdacc1'
+  uploader.base_html = "index.php"
+  body = uploader.get_base
+  each_filename(body) do |filename|
+    exist = uploader.local_file_exist?(filename)
+    puts [ filename, exist ? 'exist' : 'download' ].join("\t")
+    uploader.download_file_directly(filename, '/log') unless exist
+  end

diff --git a/bin/mmdacc2-idx.rb b/bin/mmdacc2-idx.rb
new file mode 100755 (executable)
index 0000000..7117149
--- /dev/null
+++ b/bin/mmdacc2-idx.rb
@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+# download from mikumikudance
+# http://www9.atpages.jp/~mmdaccessory/uploader2/index.html
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_filename(text)
+  file_href = 'http://www9.atpages.jp/~mmdaccessory/uploader2/src/'
+  file_re = Regexp.new(Regexp.escape(file_href) + '(.+)\.html')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "www9.atpages.jp"
+uploader.root_path = "/~mmdaccessory/uploader2"
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/mmdacc2'
+  uploader.base_html = "index.html"
+  body = uploader.get_base
+  each_filename(body) do |filename|
+    exist = uploader.local_file_exist?(filename)
+    puts [ filename, exist ? 'exist' : 'download' ].join("\t")
+    uploader.download_file_through_counter(filename) unless exist
+  end

diff --git a/bin/mmdfile-idx.rb b/bin/mmdfile-idx.rb
new file mode 100755 (executable)
index 0000000..a3f57fc
--- /dev/null
+++ b/bin/mmdfile-idx.rb
@@ -0,0 +1,25 @@
+#!/usr/bin/ruby
+# download from mikumikudance
+# http://loda.jp/mmdfile/
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_filename(text)
+  text.scan(%r(<a href="(.+?)" >(.+?)</a>)) do |href, name |
+    yield name
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "loda.jp"
+uploader.root_path = "/mmdfile"
+uploader.local_dir = '/Volumes/uploader/arc/mmdfile'
+  uploader.base_html = "/?page=1"
+  body = uploader.get_base
+  open('/Volumes/uploader/src/mmdfile/1.html', 'w') { |o| o.puts body }
+  each_filename(body) do |filename|
+    exist = uploader.local_file_exist?(filename)
+    puts [ filename, exist ? 'exist' : 'download' ].join("\t")
+    uploader.download_file_directly_id(filename)
+  end

diff --git a/bin/mod-idx.rb b/bin/mod-idx.rb
new file mode 100755 (executable)
index 0000000..a736f96
--- /dev/null
+++ b/bin/mod-idx.rb
@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+# download from futabacustom mod
+# http://www.nijibox5.com/futabacustom/mod/
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_filename(text)
+  file_href = './src/'
+  file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "www.nijibox5.com"
+uploader.root_path = "/futabacustom/mod"
+uploader.base_html = ""
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/mod'
+body = uploader.get_base
+each_filename(body) do |filename|
+  exist = uploader.local_file_exist?(filename)
+  puts [ filename, exist ? 'exist' : 'download' ].join("\t")
+  uploader.download_file(filename) unless exist
+end

diff --git a/bin/pnt-idx.rb b/bin/pnt-idx.rb
new file mode 100755 (executable)
index 0000000..f8a39b2
--- /dev/null
+++ b/bin/pnt-idx.rb
@@ -0,0 +1,28 @@
+#!/usr/bin/ruby\r
+# download from futabacustom pose\r
+# http://www.nijibox5.com/futabacustom/pose/\r
+\r
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')\r
+require 'sn_uploader'\r
+\r
+def each_filename(text)\r
+  file_href = './src/'\r
+  file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html')\r
+  text.scan(/href="(.+?)"/) do |href, |\r
+    if md = file_re.match(href)\r
+      yield md[1]\r
+    end\r
+  end\r
+end\r
+\r
+uploader = SnUploader.new\r
+uploader.host = "www.nijibox5.com"\r
+uploader.root_path = "/futabacustom/pose"\r
+uploader.base_html = ""\r
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/pnt'\r
+body = uploader.get_base\r
+each_filename(body) do |filename|\r
+  exist = uploader.local_file_exist?(filename)\r
+  puts [ filename, exist ? 'exist' : 'download' ].join("\t")\r
+  uploader.download_file(filename) unless exist\r
+end\r

diff --git a/bin/tap-idx.rb b/bin/tap-idx.rb
new file mode 100755 (executable)
index 0000000..0a88e26
--- /dev/null
+++ b/bin/tap-idx.rb
@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+# download from 3ch pose
+# http://www.esc-j.net/tech-arts/pose/
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_filename(text)
+  file_href = 'http://www.esc-j.net/tech-arts/pose/src/'
+  file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "www.esc-j.net"
+uploader.root_path = "/tech-arts/pose"
+uploader.base_html = ""
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/tap'
+body = uploader.get_base
+each_filename(body) do |filename|
+  exist = uploader.local_file_exist?(filename)
+  puts [ filename, exist ? 'exist' : 'download' ].join("\t")
+  uploader.download_file(filename) unless exist
+end

diff --git a/bin/tim-idx.rb b/bin/tim-idx.rb
new file mode 100755 (executable)
index 0000000..a75f065
--- /dev/null
+++ b/bin/tim-idx.rb
@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+# download from futabacustom time
+# http://www.nijibox5.com/futabacustom/time/
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_filename(text)
+  file_href = './src/'
+  file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "www.nijibox5.com"
+uploader.root_path = "/futabacustom/time"
+uploader.base_html = ""
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/tim'
+body = uploader.get_base
+each_filename(body) do |filename|
+  exist = uploader.local_file_exist?(filename)
+  puts [ filename, exist ? 'exist' : 'download' ].join("\t")
+  uploader.download_file(filename) unless exist
+end

diff --git a/bin/wpc-idx.rb b/bin/wpc-idx.rb
new file mode 100755 (executable)
index 0000000..8a2f6f2
--- /dev/null
+++ b/bin/wpc-idx.rb
@@ -0,0 +1,30 @@
+#!/usr/bin/ruby
+# download from 3DCG Craftsmen's Guild
+# http://3dcustom.ath.cx/wordpress/wp-content/uploader/upload.html
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_fileno(text)
+  file_href = './upload.cgi?mode=dl&file='
+  file_re = Regexp.new(Regexp.escape(file_href) + '(\d+)')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "3dcustom.ath.cx"
+uploader.root_path = "/uploader/mod"
+uploader.base_html = "upload.html"
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/wpc'
+body = uploader.get_base
+each_fileno(body) do |fileno|
+  basename_without_extension = "MODS%04d" % fileno.to_i
+  filename_re = Regexp.new('\A' + Regexp.escape(basename_without_extension) + '\.')
+  exist = uploader.local_file_match?(filename_re)
+  puts [ fileno, exist ? 'exist' : 'download' ].join("\t")
+  uploader.download(fileno) unless exist
+end

diff --git a/bin/xpc-idx.rb b/bin/xpc-idx.rb
new file mode 100755 (executable)
index 0000000..ba68b1a
--- /dev/null
+++ b/bin/xpc-idx.rb
@@ -0,0 +1,30 @@
+#!/usr/bin/ruby
+# download from 3dcustom.net
+# http://3dcustom.net/TACuploader/upload.html
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_fileno(text)
+  file_href = './upload.cgi?mode=dl&file='
+  file_re = Regexp.new(Regexp.escape(file_href) + '(\d+)')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "up1.3dcustom.net"
+uploader.root_path = "/TACuploader"
+uploader.base_html = "upload.html"
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/xpc'
+body = uploader.get_base
+each_fileno(body) do |fileno|
+  basename_without_extension = "XPC%05d" % fileno.to_i
+  filename_re = Regexp.new('\A' + Regexp.escape(basename_without_extension) + '\.')
+  exist = uploader.local_file_match?(filename_re)
+  puts [ fileno, exist ? 'exist' : 'download' ].join("\t")
+  uploader.download(fileno) unless exist
+end

diff --git a/bin/xpp-idx.rb b/bin/xpp-idx.rb
new file mode 100755 (executable)
index 0000000..546763f
--- /dev/null
+++ b/bin/xpp-idx.rb
@@ -0,0 +1,27 @@
+#!/usr/bin/ruby
+# download from 3dcustom.net pose
+#
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'sn_uploader'
+
+def each_filename(text)
+  file_href = './src/'
+  file_re = Regexp.new(Regexp.escape(file_href) + '(.+?)\.html')
+  text.scan(/href="(.+?)"/) do |href, |
+    if md = file_re.match(href)
+      yield md[1]
+    end
+  end
+end
+
+uploader = SnUploader.new
+uploader.host = "3dcustom.net"
+uploader.root_path = "/TAPuploader"
+uploader.base_html = "upload.html"
+uploader.local_dir = File.dirname(__FILE__) + '/../arc/xpp'
+body = uploader.get_base
+each_filename(body) do |filename|
+  exist = uploader.local_file_exist?(filename)
+  puts [ filename, exist ? 'exist' : 'download' ].join("\t")
+  uploader.download_file(filename) unless exist
+end

diff --git a/lib/sn_uploader.rb b/lib/sn_uploader.rb
new file mode 100755 (executable)
index 0000000..34e303b
--- /dev/null
+++ b/lib/sn_uploader.rb
@@ -0,0 +1,170 @@
+require 'net/http'
+require 'erb'
+require 'nkf'
+require 'user_agent'
+
+class SnUploader
+  attr_accessor :host, :root_path, :jcode, :dlkey, :authorization, :local_dir, :base_html
+
+  def initialize
+    @jcode = ERB::Util.u("\8a¿\8e\9a")
+    @dlkey = "custom"
+    @authorization = nil
+    @local_dir = "."
+    @base_html = "upload.html"
+  end
+
+  def kcode=(kcode)
+    case kcode
+    when /^U/i
+      @jcode = ERB::Util.u(NKF.nkf('-Sw', "\8a¿\8e\9a"))
+    end
+  end
+
+  def header
+    header = UserAgent.default_http_header.dup
+    header["Authorization"] = authorization if authorization
+    header
+  end
+
+  def local_path(path)
+    File.join(local_dir, File.basename(path))
+  end
+
+  def local_file_exist?(filename)
+    File.exist?(local_path(filename))
+  end
+
+  def local_file_match?(filename_re)
+    Dir.foreach(local_dir) do |ent|
+      if md = filename_re.match(ent)
+        return md[0]
+      end
+    end
+    false
+  end
+
+  def download(fileno, filename_re = nil)
+    if filename_re && local_file_exist?(filename_re)
+      return nil
+    end
+
+    Net::HTTP.new(host).start do |http|
+      response = http.post("#{root_path}/upload.cgi", "file=#{fileno}&jcode=#{jcode}&mode=dl&dlkey=#{dlkey}", header)
+      body = response.body
+
+      if md = /"1;URL=(.+?)"/.match(body)
+        path = md[1]
+        path.sub!(/\A\./, root_path)
+      end
+      puts path
+      if path
+        open(local_path(path), "wb") do |file|
+          response = http.get(path, header) do |data|
+            file.write data
+          end
+        end
+      end
+    end
+  end
+
+  def download_file_through_counter(filename)
+    if local_file_exist?(filename)
+      return nil
+    end
+
+    Net::HTTP.new(host).start do |http|
+      response = http.get("#{root_path}/src/#{filename}.html", header)
+      file_re = Regexp.new(Regexp.escape('count.cgi?') + '(.+)')
+      text = response.body
+      text.scan(/href="(.+?)"/) do |href, |
+        if md = file_re.match(href)
+          path = root_path + '/src/' + md[1]
+          open(local_path(filename), "wb") do |file|
+            response = http.get(path, header) do |data|
+              file.write data
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def download_file(filename)
+    if local_file_exist?(filename)
+      return nil
+    end
+
+    Net::HTTP.new(host).start do |http|
+      response = http.get("#{root_path}/src/#{filename}.html", header)
+      body = response.body
+      if md = /href="(.+?)"/.match(body)
+        path = md[1]
+        path.sub!(/\A\./, root_path + "/src")
+      end
+      puts path
+      if path
+        open(local_path(path), "wb") do |file|
+          response = http.get(path, header) do |data|
+            file.write data
+          end
+        end
+      end
+    end
+  end
+
+  def download_file_directly(filename, dir = '/src')
+    if local_file_exist?(filename)
+      return nil
+    end
+
+    path = "#{root_path}#{dir}/#{filename}"
+    puts path
+    Net::HTTP.new(host).start do |http|
+      open(local_path(path), "wb") do |file|
+        response = http.get(path, header) do |data|
+          file.write data
+        end
+      end
+    end
+  end
+
+  def download_file_directly_id(filename)
+    if local_file_exist?(filename)
+      return nil
+    end
+
+    path = "#{root_path}/?id=#{filename}"
+    puts path
+    http = Net::HTTP.new(host)
+    http.open_timeout = 10
+    http.read_timeout = 20
+    begin
+      http.start do
+        open(local_path(filename), "wb") do |file|
+          response = http.get(path, header) do |data|
+            file.write data
+          end
+        end
+      end
+    rescue Timeout::Error
+      puts "timeout..."
+      sleep(20)
+      puts "retry"
+      retry
+    end
+  end
+
+  def get_base
+    body = nil
+    Net::HTTP.new(host).start do |http|
+      response = http.get("#{root_path}/#{base_html}", header)
+      # response.each do |name, value|
+      #   puts [name, value].join(': ')
+      # end
+      # puts
+      body = response.body
+    end
+    body
+  end
+end

diff --git a/lib/user_agent.rb b/lib/user_agent.rb
new file mode 100755 (executable)
index 0000000..b3cf55a
--- /dev/null
+++ b/lib/user_agent.rb
@@ -0,0 +1,11 @@
+module UserAgent
+module_function
+  def default_http_header
+    @_default_http_header ||= {
+      "User-Agent" => "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10 (.NET CLR 3.5.30729)",
+      "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "Accept-Language" => "ja,en-us;q=0.7,en;q=0.3",
+      "Accept-Charset" => "Shift_JIS,utf-8;q=0.7,*;q=0.7",
+    }
+  end
+end