karchnu/mechanize.cr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

format code

Browse Source
This commit is contained in:
Anton Maminov 2021-08-17 19:30:01 +03:00
parent 858a999ed2
commit 9478bffdf1
29 changed files with 153 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
main.cr
View File

@ -4,10 +4,10 @@ agent = Mechanize.new
agent.request_headers = HTTP::Headers{"Foo" => "Bar"}
params = {"hoge" => "hoge"}
page = agent.get("http://example.com/", params: params)
#form = page.forms[0]
#query = {"foo" => "foo_value", "bar" => "bar_value"}
#page = agent.post("http://example.com/", query: query)
#puts page.code
#puts page.body
#puts page.css("h1").first.inner_text
#puts page.title
# form = page.forms[0]
# query = {"foo" => "foo_value", "bar" => "bar_value"}
# page = agent.post("http://example.com/", query: query)
# puts page.code
# puts page.body
# puts page.css("h1").first.inner_text
# puts page.title

11
spec/cookie_spec.cr
View File

@ -6,12 +6,11 @@ WebMock.stub(:get, "example.com/cookies2").to_return(headers: {"Set-Cookie" => "
WebMock.stub(:get, "example.com/cookies3").to_return(headers: {"Set-Cookie" => "id=456"})
WebMock.stub(:get, "example.com/secure_cookies").to_return(headers: {"Set-Cookie" => "id=123; Secure"})
WebMock.stub(:get, "example.com/paths").to_return(headers: {"Set-Cookie" => "id=123; Path=/paths"})
WebMock.stub(:get, "example.com/paths/hoge").to_return()
WebMock.stub(:get, "https://example.com/").to_return()
WebMock.stub(:get, "example.com/hoge/paths").to_return()
WebMock.stub(:get, "www.example.com").to_return()
WebMock.stub(:get, "example.com/meta_cookie").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/paths/hoge").to_return
WebMock.stub(:get, "https://example.com/").to_return
WebMock.stub(:get, "example.com/hoge/paths").to_return
WebMock.stub(:get, "www.example.com").to_return
WebMock.stub(:get, "example.com/meta_cookie").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

3
spec/form/button_spec.cr
View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/button").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/button").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

3
spec/form/check_box_spec.cr
View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/check_box").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/check_box").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

3
spec/form/field_spec.cr
View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/fields").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/fields").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

3
spec/form/multi_select_list_spec.cr
View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/multi_select_list").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/multi_select_list").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

3
spec/form/option_spec.cr
View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/multi_select_list").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/multi_select_list").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

5
spec/form/radio_button_spec.cr
View File

@ -1,7 +1,6 @@
require "../spec_helper.cr"
WebMock.stub(:get, "example.com/form/radio_button").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/radio_button").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>
@ -26,7 +25,7 @@ describe "Form Fields RadioButton" do
radiobuttons.size.should eq 3
it "returns radiobutton check status" do
radiobuttons.map(&.checked?).should eq [false,false,false]
radiobuttons.map(&.checked?).should eq [false, false, false]
end
it "can change check status" do

3
spec/form/select_list_spec.cr
View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/select_list").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/select_list").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

3
spec/form/textarea_spec.cr
View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/textarea").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/textarea").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

3
spec/form_spec.cr
View File

@ -1,7 +1,6 @@
require "./spec_helper"
WebMock.stub(:get, "example.com/check_form").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/check_form").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

8
spec/http_spec.cr
View File

@ -1,8 +1,8 @@
require "./spec_helper"
WebMock.stub(:get, "http://example.com/?foo=bar&foo1=bar2")
WebMock.stub(:post, "http://example.com/post").
with(body: "email=foobar", headers: {"Content-Type" => "application/x-www-form-urlencoded"}).
to_return(body: "success")
WebMock.stub(:post, "http://example.com/post")
.with(body: "email=foobar", headers: {"Content-Type" => "application/x-www-form-urlencoded"})
.to_return(body: "success")
WebMock.stub(:get, "example.com/%E3%81%82%E3%81%82%E3%81%82")
describe "Mechanize HTTP test" do
@ -49,7 +49,7 @@ describe "Mechanize HTTP test" do
it "simple POST" do
agent = Mechanize.new
query = { "email" => "foobar" }
query = {"email" => "foobar"}
page = agent.post("http://example.com/post", query: query)
page.body.should eq "success"
page.code.should eq 200

4
spec/page_spec.cr
View File

@ -35,14 +35,14 @@ describe "Mechanize Page test" do
it "can detect form by using form_with method, argument type: Hash" do
agent = Mechanize.new
page = agent.get("http://example.com/form")
form = page.form_with({"name" => "sample_form" })
form = page.form_with({"name" => "sample_form"})
form.name.should eq "sample_form"
end
it "can detect form by using form_with method, argument type: NamedTuple" do
agent = Mechanize.new
page = agent.get("http://example.com/form")
form = page.form_with({name: "sample_form" })
form = page.form_with({name: "sample_form"})
form.name.should eq "sample_form"
end
end

16
spec/spec_helper.cr
View File

@ -2,14 +2,12 @@ require "spec"
require "webmock"
require "../src/mechanize"
WebMock.stub(:get, "example.com")
WebMock.stub(:get, "fail_example.com").to_return(status: 500)
WebMock.stub(:get, "body_example.com").to_return(body: "hello")
WebMock.stub(:get, "another_domain.com/")
WebMock.stub(:get, "example.com/form").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>
@ -24,10 +22,10 @@ WebMock.stub(:get, "example.com/form").to_return(body:
</html>
BODY
)
WebMock.stub(:post, "example.com/post_path").
with(body: "name=foo&email=bar", headers: {"Content-Type" => "application/x-www-form-urlencoded"}).
to_return(body: "success")
WebMock.stub(:post, "example.com/post_path")
.with(body: "name=foo&email=bar", headers: {"Content-Type" => "application/x-www-form-urlencoded"})
.to_return(body: "success")
WebMock.stub(:post, "example.com/post_path").
with(body: "name=foo&email=bar&commit=submit", headers: {"Content-Type" => "application/x-www-form-urlencoded"}).
to_return(body: "success with button")
WebMock.stub(:post, "example.com/post_path")
.with(body: "name=foo&email=bar&commit=submit", headers: {"Content-Type" => "application/x-www-form-urlencoded"})
.to_return(body: "success with button")

19
src/mechanize.cr
View File

@ -10,27 +10,28 @@ class Mechanize
AGENT = {
"Mechanize" => "Mechanize/#{VERSION} Crystal/#{Crystal::VERSION} (https://github.com/Kanezoh/mechanize.cr)",
}
def initialize()
def initialize
@agent = MechanizeCr::HTTP::Agent.new
@agent.context = self
@agent.user_agent = AGENT["Mechanize"]
end
def get(uri : String | URI, headers = HTTP::Headers.new, params : Hash(String, String | Array(String)) = Hash(String,String).new)
def get(uri : String | URI, headers = HTTP::Headers.new, params : Hash(String, String | Array(String)) = Hash(String, String).new)
method = :get
page = @agent.fetch uri, method, headers, params
add_to_history(page)
#yield page if block_given?
# yield page if block_given?
page
end
def post(uri : String | URI, headers = HTTP::Headers.new, query : Hash(String, String | Array(String)) = Hash(String,String).new)
def post(uri : String | URI, headers = HTTP::Headers.new, query : Hash(String, String | Array(String)) = Hash(String, String).new)
node = Node.new
node["method"] = "POST"
node["enctype"] = "application/x-www-form-urlencoded"
form = MechanizeCr::Form.new(node)
query.each do |k,v|
query.each do |k, v|
node = Node.new
node["name"] = k
form.fields << MechanizeCr::FormContent::Field.new(node, v)
@ -44,13 +45,13 @@ class Mechanize
request_data = form.request_data
content_headers = ::HTTP::Headers{
"Content-Type" => form.enctype,
"Content-Length" => request_data.size.to_s,
"Content-Type" => form.enctype,
"Content-Length" => request_data.size.to_s,
}
headers.merge!(content_headers)
# fetch the page
page = @agent.fetch(uri, :post, headers: headers, params: {"value" => request_data }, referer: cur_page)
page = @agent.fetch(uri, :post, headers: headers, params: {"value" => request_data}, referer: cur_page)
headers.delete("Content-Type")
headers.delete("Content-Length")
add_to_history(page)
@ -81,7 +82,7 @@ class Mechanize
@agent.history.pop
end
def submit(form, button=nil)
def submit(form, button = nil)
form.add_button_to_query(button) if button
case form.method.upcase
when "POST"

11
src/mechanize/cookie.cr
View File

@ -6,11 +6,12 @@
# this property is used to send cookies to same origin resource.
class ::HTTP::Cookie
property origin : String?
def initialize(name : String, value : String, @path : String? = nil,
@expires : Time? = nil, @domain : String? = nil,
@secure : Bool = false, @http_only : Bool = false,
@samesite : SameSite? = nil, @extension : String? = nil,
@origin : String? = nil)
@expires : Time? = nil, @domain : String? = nil,
@secure : Bool = false, @http_only : Bool = false,
@samesite : SameSite? = nil, @extension : String? = nil,
@origin : String? = nil)
validate_name(name)
@name = name
validate_value(value)
@ -30,7 +31,7 @@ class ::HTTP::Cookie
end
if domain
host.try &.=~(/.*#{domain.try &.gsub(".", "\.")}$/)
host.try &.=~(/.*#{domain.try &.gsub(".", ".")}$/)
else
origin == host
end

4
src/mechanize/errors/element_not_found_error.cr
View File

@ -1,13 +1,13 @@
require "./base_error"
class MechanizeCr::ElementNotFoundError < MechanizeCr::Error
getter element : Symbol
getter conditions : String
def initialize(element, conditions)
@element = element
@element = element
@conditions = conditions
super "Element #{element} with conditions #{conditions} was not found"
end
end

8
src/mechanize/file.cr
View File

@ -1,9 +1,11 @@
require "http/client"
class MechanizeCr::File
#property :body, :filename
# property :body, :filename
property :body, :code, uri, :response
def initialize(uri : URI, response : ::HTTP::Client::Response, body : String , code : Int32)
@uri = uri
def initialize(uri : URI, response : ::HTTP::Client::Response, body : String, code : Int32)
@uri = uri
@body = body
@code = code
end

64
src/mechanize/form.cr
View File

@ -11,35 +11,35 @@ require "./utils/element_matcher"
class MechanizeCr::Form
include MechanzeCr::ElementMatcher
getter node : Node | Lexbor::Node
getter fields : Array(FormContent::Field)
getter checkboxes : Array(FormContent::CheckBox)
getter node : Node | Lexbor::Node
getter fields : Array(FormContent::Field)
getter checkboxes : Array(FormContent::CheckBox)
getter radiobuttons : Array(FormContent::RadioButton)
getter selectboxes : Array(FormContent::MultiSelectList)
getter buttons : Array(FormContent::Button)
getter enctype : String
getter method : String
getter name : String
getter page : Page?
property action : String
getter selectboxes : Array(FormContent::MultiSelectList)
getter buttons : Array(FormContent::Button)
getter enctype : String
getter method : String
getter name : String
getter page : Page?
property action : String
def initialize(node : Node | Lexbor::Node, page : Page? = nil)
@enctype = node.fetch("enctype", "application/x-www-form-urlencoded")
@node = node
@fields = Array(FormContent::Field).new
@checkboxes = Array(FormContent::CheckBox).new
@radiobuttons = Array(FormContent::RadioButton).new
@selectboxes = Array(FormContent::MultiSelectList).new
@buttons = Array(FormContent::Button).new
@action = node.fetch("action", "")
@method = node.fetch("method", "GET").upcase
@name = node.fetch("name", "")
@clicked_buttons = Array(FormContent::Button).new
@page = page
#@mech = mech
@enctype = node.fetch("enctype", "application/x-www-form-urlencoded")
@node = node
@fields = Array(FormContent::Field).new
@checkboxes = Array(FormContent::CheckBox).new
@radiobuttons = Array(FormContent::RadioButton).new
@selectboxes = Array(FormContent::MultiSelectList).new
@buttons = Array(FormContent::Button).new
@action = node.fetch("action", "")
@method = node.fetch("method", "GET").upcase
@name = node.fetch("name", "")
@clicked_buttons = Array(FormContent::Button).new
@page = page
# @mech = mech
#@encoding = node['accept-charset'] || (page && page.encoding) || nil
#@ignore_encoding_error = false
# @encoding = node['accept-charset'] || (page && page.encoding) || nil
# @ignore_encoding_error = false
parse
end
@ -75,7 +75,7 @@ class MechanizeCr::Form
buttons << FormContent::Button.new(html_node, @node)
when "submit"
buttons << FormContent::SubmitButton.new(html_node, @node)
when"reset"
when "reset"
buttons << FormContent::ResetButton.new(html_node, @node)
when "image"
buttons << FormContent::ImageButton.new(html_node, @node)
@ -118,7 +118,7 @@ class MechanizeCr::Form
private def build_query_string(params : Array(Array(String)))
params.reduce("") do |acc, arr|
hash = { arr[0] => arr[1] }
hash = {arr[0] => arr[1]}
acc + URI::Params.encode(hash) + '&'
end.rchop
end
@ -144,9 +144,9 @@ class MechanizeCr::Form
radio_groups.each_value do |g|
checked = g.select(&.checked)
if checked.uniq.size > 1
#values = checked.map(&.value).join(', ').inspect
#name = checked.first.name.inspect
#raise Mechanize::Error,
# values = checked.map(&.value).join(', ').inspect
# name = checked.first.name.inspect
# raise Mechanize::Error,
# "radiobuttons #{values} are checked in the #{name} group, " \
# "only one is allowed"
raise MechanizeCr::Error.new
@ -182,8 +182,8 @@ class MechanizeCr::Form
def add_button_to_query(button)
unless button.form_node == @node
message = ""
"#{button.inspect} does not belong to the same page as " \
"the form #{@name.inspect} in #{@page.try &.uri}"
"#{button.inspect} does not belong to the same page as " \
"the form #{@name.inspect} in #{@page.try &.uri}"
message = "not a valid button"
raise ArgumentError.new(message)
end

4
src/mechanize/form/button.cr
View File

@ -1,10 +1,12 @@
class MechanizeCr::FormContent::Button < MechanizeCr::FormContent::Field
getter form_node : Node | Lexbor::Node
def initialize(node : Node | Lexbor::Node, form_node : Node | Lexbor::Node, value=nil)
def initialize(node : Node | Lexbor::Node, form_node : Node | Lexbor::Node, value = nil)
@form_node = form_node
super(node, value)
end
end
require "./reset_button"
require "./submit_button"
require "./image_button"

1
src/mechanize/form/check_box.cr
View File

@ -2,6 +2,7 @@ class MechanizeCr::FormContent::CheckBox < MechanizeCr::FormContent::RadioButton
def check
@checked = true
end
def query_value
[@name, @value || "on"]
end

20
src/mechanize/form/field.cr
View File

@ -1,15 +1,15 @@
class MechanizeCr::FormContent::Field
property value : String?
getter name : String
getter type : String
property value : String?
getter name : String
getter type : String
getter raw_value : String?
getter node : Node | Lexbor::Node
getter node : Node | Lexbor::Node
def initialize(node : Node | Lexbor::Node, value=nil)
@node = node
@name = node.fetch("name", "")
@value = value || node.fetch("value", nil)
@type = node.fetch("type", "")
def initialize(node : Node | Lexbor::Node, value = nil)
@node = node
@name = node.fetch("name", "")
@value = value || node.fetch("value", nil)
@type = node.fetch("type", "")
@raw_value = @value
end
@ -30,7 +30,7 @@ class MechanizeCr::FormContent::Field
def inspect # :nodoc:
"[%s:0x%x type: %s name: %s value: %s]" % [
self.class.name.sub(/MechanizeCr::FormContent::/, "").downcase,
object_id, type, name, value
object_id, type, name, value,
]
end
end

20
src/mechanize/form/multi_select_list.cr
View File

@ -1,17 +1,17 @@
require "./option"
class MechanizeCr::FormContent::MultiSelectList
getter node : Lexbor::Node
getter name : String
getter type : String
property values : Array(String)
property options : Array(FormContent::Option)
getter node : Lexbor::Node
getter name : String
getter type : String
property values : Array(String)
property options : Array(FormContent::Option)
def initialize(node : Lexbor::Node)
@node = node
@name = node.fetch("name", "")
@type = node.fetch("type", "")
@values = Array(String).new
@node = node
@name = node.fetch("name", "")
@type = node.fetch("type", "")
@values = Array(String).new
@options = Array(FormContent::Option).new
node.css("option").each { |n|
@options << FormContent::Option.new(n, self)
@ -55,7 +55,7 @@ class MechanizeCr::FormContent::MultiSelectList
def inspect # :nodoc:
"[%s:0x%x type: %s name: %s values: [%s]]" % [
self.class.name.sub(/MechanizeCr::FormContent::/, "").downcase,
object_id, type, name, values.join(',')
object_id, type, name, values.join(','),
]
end
end

16
src/mechanize/form/option.cr
View File

@ -1,15 +1,15 @@
class MechanizeCr::FormContent::Option
getter select_list : FormContent::MultiSelectList
getter node : Lexbor::Node
getter text : String
getter value : String
getter selected : Bool
getter node : Lexbor::Node
getter text : String
getter value : String
getter selected : Bool
def initialize(node, select_list)
@node = node
@text = node.inner_text
@value = (node["value"] || node.inner_text)
@selected = node.has_key?("selected")
@node = node
@text = node.inner_text
@value = (node["value"] || node.inner_text)
@selected = node.has_key?("selected")
@select_list = select_list # The select list this option belongs to
end

33
src/mechanize/form/radio_button.cr
View File

@ -3,7 +3,7 @@ class MechanizeCr::FormContent::RadioButton < MechanizeCr::FormContent::Field
def initialize(node : Node | Lexbor::Node, form : Form)
@checked = !!node.fetch("checked", nil)
@form = form
@form = form
super(node)
end
@ -24,33 +24,32 @@ class MechanizeCr::FormContent::RadioButton < MechanizeCr::FormContent::Field
checked
end
#def hash # :nodoc:
# def hash # :nodoc:
# @form.hash ^ @name.hash ^ @value.hash
#end
#
#def label
# end
#
# def label
# (id = self['id']) && @form.page.labels_hash[id] || nil
#end
#
#def text
# end
#
# def text
# label.text rescue nil
#end
#
#def [](key)
# end
#
# def [](key)
# @node[key]
#end
# end
# alias checked? checked
#def == other # :nodoc:
# def == other # :nodoc:
# self.class === other and
# other.form == @form and
# other.name == @name and
# other.value == @value
#end
#
#alias eql? == # :nodoc:
# end
#
# alias eql? == # :nodoc:
private def uncheck_peers
form.radiobuttons_with(name).try &.each do |b|

1
src/mechanize/history.cr
View File

@ -2,6 +2,7 @@ require "./page"
class MechanizeCr::History < Array(MechanizeCr::Page)
property max_size : Int32
def initialize(max_size = 100)
@max_size = max_size
super

12
src/mechanize/http/agent.cr
View File

@ -19,7 +19,7 @@ module MechanizeCr
@user_agent = ""
end
def fetch(uri, method = :get, headers = HTTP::Headers.new, params = Hash(String,String).new,
def fetch(uri, method = :get, headers = HTTP::Headers.new, params = Hash(String, String).new,
referer = (current_page unless history.empty?))
uri = resolve_url(uri, referer)
set_request_headers(uri, headers)
@ -52,7 +52,7 @@ module MechanizeCr
def http_request(uri, method, params)
case uri.scheme.not_nil!.downcase
when "http", "https" then
when "http", "https"
case method
when :get
::HTTP::Client.get(uri, headers: request_headers)
@ -82,7 +82,7 @@ module MechanizeCr
private def set_request_headers(uri, headers)
reset_request_header_cookies
headers.each do |k,v|
headers.each do |k, v|
request_headers[k] = v
end
valid_cookies(uri).add_request_headers(request_headers)
@ -117,8 +117,8 @@ module MechanizeCr
private def save_response_cookies(response, uri, page)
if page.body =~ /Set-Cookie/
page.css("head meta[http-equiv=\"Set-Cookie\"]").each do |meta|
cookie = meta["content"].split(";")[0]
key,value = cookie.split("=")
cookie = meta["content"].split(";")[0]
key, value = cookie.split("=")
cookie = ::HTTP::Cookie.new(name: key, value: value)
save_cookies(uri, [cookie])
end
@ -148,7 +148,7 @@ module MechanizeCr
end
# fill host if host isn't set
if target_url.host.nil? && referer && referer_uri.try &.host
if target_url.host.nil? && referer && referer_uri.try &.host
target_url.host = referer_uri.not_nil!.host
end
# fill scheme if scheme isn't set

4
src/mechanize/node.cr
View File

@ -1,9 +1,9 @@
require "lexbor"
# This is a fake node used when sending post request.
class Node < Hash(String,String)
class Node < Hash(String, String)
def css(str)
[] of Hash(String,String)
[] of Hash(String, String)
end
def inner_text

2
src/mechanize/utils/element_matcher.cr
View File

@ -1,5 +1,5 @@
module MechanzeCr::ElementMatcher
macro elements_with(singular, plural="")
macro elements_with(singular, plural = "")
{% plural = "#{singular.id}s" if plural.empty? %}
def {{plural.id}}_with(criteria)
{{plural.id}}_with(criteria){}