README.md

@@ -4,8 +4,7 @@
 
 This project is inspired by Ruby's [mechanize](https://github.com/sparklemotion/mechanize).
 The purpose is to cover all the features of original one.
-
+Now, mechanize.cr can automatically store and send cookies, fill and submit forms.
-[API Documentation](https://kanezoh.github.io/mechanize.cr/)
 
 ## Installation
 
@@ -21,7 +20,7 @@ The purpose is to cover all the features of original one.
 
 ## Usage
 
-### GET request
+### simple GET request
 
 ```crystal
 require "mechanize"
@@ -96,17 +95,6 @@ For activation, simply setup the log to `:debug` level
 Log.setup("mechanize", :debug)
 ```
 
-### Basic auth
-
-You can access a page which is protected by basic auth, setting username and password for the url.
-
-```crystal
-agent = Mechanize.new
-agent.add_auth("http://example.com", "username", "password")
-agent.get("http://example.com")
-```
-
-
 ## Contributing
 
 1. Fork it (<https://github.com/Kanezoh/mechanize.cr/fork>)

main.cr

@@ -0,0 +1,13 @@
+require "./src/mechanize.cr"
+
+agent = Mechanize.new
+agent.request_headers = HTTP::Headers{"Foo" => "Bar"}
+params = {"hoge" => "hoge"}
+page = agent.get("http://example.com/", params: params)
+# form = page.forms[0]
+# query = {"foo" => "foo_value", "bar" => "bar_value"}
+# page = agent.post("http://example.com/", query: query)
+# puts page.code
+# puts page.body
+# puts page.css("h1").first.inner_text
+# puts page.title

shard.yml

@@ -11,8 +11,6 @@ license: MIT
 dependencies:
   lexbor:
     github: kostya/lexbor
-  http_proxy:
-    github: mamantoha/http_proxy
 
 development_dependencies:
   webmock:
@@ -20,6 +18,6 @@ development_dependencies:
     branch: master
   kemal:
     github: kemalcr/kemal
-    version: ~> 1.3.0
+    version: ~> 1.0.0
   kemal-basic-auth:
     github: kemalcr/kemal-basic-auth

spec/http_spec.cr

@@ -1,6 +1,5 @@
 require "./spec_helper"
 WebMock.stub(:get, "http://example.com/?foo=bar&foo1=bar2")
-WebMock.stub(:get, "http://example.com/path?foo=bar&foo1=bar2")
 WebMock.stub(:post, "http://example.com/post")
   .with(body: "email=foobar", headers: {"Content-Type" => "application/x-www-form-urlencoded"})
   .to_return(body: "success")
@@ -41,14 +40,6 @@ describe "Mechanize HTTP test" do
     page.uri.to_s.should eq uri
   end
 
-  it "GET with query parameter with path" do
-    agent = Mechanize.new
-    uri = "http://example.com/path?foo=bar&foo1=bar2"
-    page = agent.get(uri)
-    page.code.should eq 200
-    page.uri.to_s.should eq uri
-  end
-
   it "simple POST" do
     agent = Mechanize.new
     query = {"email" => "foobar"}

spec/proxy_spec.cr

@@ -1,17 +0,0 @@
-require "./spec_helper"
-
-WebMock.stub(:get, "http://example.com/with_proxy").to_return(body: "success")
-
-describe "Mechanize proxy test" do
-  it "set proxy" do
-    with_proxy_server do |host, port, wants_close|
-      agent = Mechanize.new
-      agent.set_proxy("127.0.0.1", 8080)
-      page = agent.get("http://example.com/with_proxy")
-      page.body.should eq("success")
-      page.code.should eq(200)
-    ensure
-      wants_close.send(nil)
-    end
-  end
-end

spec/spec_helper.cr

@@ -1,6 +1,5 @@
 require "spec"
 require "webmock"
-require "http_proxy"
 require "../src/mechanize"
 
 WebMock.stub(:get, "example.com")
@@ -47,22 +46,3 @@ WebMock.stub(:post, "example.com/post_path")
 WebMock.stub(:post, "example.com/post_path")
   .with(body: "name=foo&email=bar&commit=submit", headers: {"Content-Type" => "application/x-www-form-urlencoded"})
   .to_return(body: "success with button")
-
-def with_proxy_server(host = "127.0.0.1", port = 8080, &)
-  wants_close = Channel(Nil).new
-  server = HTTP::Proxy::Server.new
-
-  spawn do
-    server.bind_tcp(host, port)
-    server.listen
-  end
-
-  spawn do
-    wants_close.receive
-    server.close
-  end
-
-  Fiber.yield
-
-  yield host, port, wants_close
-end

src/mechanize.cr

@@ -2,7 +2,6 @@ require "log"
 require "uri"
 require "http/client"
 require "lexbor"
-require "http_proxy"
 require "./mechanize/http/agent"
 require "./mechanize/form"
 require "./mechanize/node"
@@ -299,18 +298,13 @@ class Mechanize
 
   # set basic auth credentials.
   # ```
-  # # set an auth credential with a specific url.
+  # # make download.html whose content is http://example.com's html.
   # agent.add_auth("http://example.com", "username", "password")
   # ```
   def add_auth(uri : String, user : String, pass : String)
     @agent.add_auth(uri, user, pass)
   end
 
-  # Sets the proxy +address+ at +port+ with an optional +user+ and +password+
-  def set_proxy(address : String, port : Int32, user : String? = nil, password : String? = nil)
-    @agent.set_proxy(address, port, user, password)
-  end
-
   # Runs given block, then resets the page history as it was before.
   private def transact
     # save the previous history status.

src/mechanize/form.cr

@@ -193,6 +193,7 @@ class Mechanize
     # submitted with multiple buttons, pass each button to this method.
     def add_button_to_query(button)
       unless button.form_node == @node
+        message = ""
         "#{button.inspect} does not belong to the same page as " \
         "the form #{@name.inspect} in #{@page.try &.uri}"
         message = "not a valid button"

src/mechanize/form/select_list.cr

@@ -23,14 +23,10 @@ class Mechanize::FormContent::SelectList < Mechanize::FormContent::MultiSelectLi
   end
 
   def value=(new_value)
-    @values = new_value
+    values = new_value
   end
 
   def query_value
-    if v = value
+    value ? [[name, value.not_nil!]] : nil
-      [[name, v]]
-    else
-      nil
-    end
   end
 end

src/mechanize/history.cr

@@ -30,7 +30,7 @@ class Mechanize
       if size == 0
         # TODO: raise error
       end
-      @array.pop
+      page = @array.pop
     end
   end
 end

src/mechanize/http/agent.cr

@@ -16,8 +16,6 @@ class Mechanize
       getter authenticate_methods : Hash(URI, Hash(String, Array(AuthRealm)))
       getter authenticate_parser : WWWAuthenticateParser
 
-      @proxy : ::HTTP::Proxy::Client?
-
       def initialize(@context : Mechanize? = nil)
         @history = History.new
         @request_headers = ::HTTP::Headers.new
@@ -45,7 +43,6 @@ class Mechanize
         uri, params = resolve_parameters(uri, method, params)
         client = ::HTTP::Client.new(uri)
         request_auth client, uri
-        client.set_proxy(@proxy) if @proxy
         response = http_request(client, uri, method, params, body)
         body = response.not_nil!.body
         page = response_parse(response, body, uri)
@@ -69,7 +66,7 @@ class Mechanize
         redirect_url = response.headers["location"]
         uri = resolve_url(redirect_url, referer)
 
-        Log.debug { "follow redirect to: #{uri}" }
+        Log.debug { "follow redirect to: #{uri.to_s}" }
 
         # Make sure we are not copying over the POST headers from the original request
         headers.delete("Content-MD5")
@@ -87,7 +84,6 @@ class Mechanize
                                body : String?) : ::HTTP::Client::Response?
         request_log(uri, method)
         path = uri.path
-        path += "?#{uri.query.not_nil!}" if uri.query
 
         case uri.scheme.not_nil!.downcase
         when "http", "https"
@@ -166,17 +162,13 @@ class Mechanize
 
       # set basic auth credentials.
       # ```
-      # # set an auth credential with a specific url.
+      # # make download.html whose content is http://example.com's html.
       # agent.add_auth("http://example.com", "username", "password")
       # ```
       def add_auth(uri : String, user : String, pass : String)
         @auth_store.add_auth(uri, user, pass)
       end
 
-      def set_proxy(address : String, port : Int32, user : String? = nil, password : String? = nil)
-        @proxy = ::HTTP::Proxy::Client.new(address, port, username: user, password: password)
-      end
-
       private def set_request_headers(uri, headers)
         reset_request_header_cookies
         headers.each do |k, v|
@@ -325,6 +317,7 @@ class Mechanize
 
       # extract valid cookies according to URI
       private def valid_cookies(uri)
+        host = uri.host
         valid_cookies = ::HTTP::Cookies.new
         request_cookies.each do |cookie|
           valid_cookies << cookie if cookie.valid_cookie?(uri)
@@ -333,7 +326,7 @@ class Mechanize
       end
 
       private def request_log(uri, method)
-        Log.debug { "#{method.to_s.upcase}: #{uri}" }
+        Log.debug { "#{method.to_s.upcase}: #{uri.to_s}" }
 
         request_headers.each do |key, values|
           value = values.size == 1 ? values.first : values

src/mechanize/http/auth_store.cr

@@ -8,7 +8,6 @@ class Mechanize
         @auth_accounts = Hash(URI, Hash(String, Array(String))).new
       end
 
-      # set an auth credential with a specific url.
       def add_auth(uri : String | URI, user : String, pass : String, realm : String? = nil, domain : String? = nil)
         target_uri = uri.dup
         unless uri.is_a?(URI)
@@ -29,6 +28,7 @@ class Mechanize
         end
       end
 
+      ##
       # Returns true if credentials exist for the +challenges+ from the server at
       # +uri+.
 

src/mechanize/http/www_authenticate_parser.cr

@@ -127,7 +127,7 @@ class Mechanize
 
         return nil unless value
 
-        [name, value]
+        return [name, value]
       end
 
       ##

src/mechanize/page.cr

@@ -43,11 +43,11 @@ class Mechanize
     # page.forms # => Array(Mechanize::Form)
     # ```
     def forms : Array(Form)
-      css("form").map do |html_form|
+      forms = css("form").map do |html_form|
         form = Form.new(html_form, self)
         form.action ||= @uri.to_s
         form
-      end
+      end.to_a
     end
 
     # return all links(`Mechanize::PageContent::Link`) in the page.
@@ -55,11 +55,11 @@ class Mechanize
     # page.links # => Array(Mechanize::PageContent::Link)
     # ```
     def links : Array(PageContent::Link)
-      %w{a area}.flat_map do |tag|
+      links = %w{a area}.map do |tag|
         css(tag).map do |node|
           PageContent::Link.new(node, @mech, self)
         end
-      end
+      end.flatten
     end
 
     elements_with "form"