Merge pull request #4 from mamantoha/format-code

Format code
master
Kanezoh 2021-08-18 14:10:12 +09:00 committed by GitHub
commit 2d6f22f8b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 167 additions and 158 deletions

View File

@ -5,6 +5,18 @@ on:
pull_request:
jobs:
check_format:
runs-on: ubuntu-latest
steps:
- name: Install Crystal
uses: crystal-lang/install-crystal@v1
- name: Check out repository code
uses: actions/checkout@v2
- name: Install dependencies
run: shards install
- name: Check format
run: crystal tool format --check
test:
strategy:
fail-fast: false

View File

@ -1,5 +1,7 @@
# mechanize.cr
[![Crystal CI](https://github.com/Kanezoh/mechanize.cr/actions/workflows/crystal.yml/badge.svg)](https://github.com/Kanezoh/mechanize.cr/actions/workflows/crystal.yml)
This project is inspired by Ruby's [mechanize](https://github.com/sparklemotion/mechanize).
The purpose is to cover all the features of original one.
Now, mechanize.cr can automatically store and send cookies, fill and submit forms.

14
main.cr
View File

@ -4,10 +4,10 @@ agent = Mechanize.new
agent.request_headers = HTTP::Headers{"Foo" => "Bar"}
params = {"hoge" => "hoge"}
page = agent.get("http://example.com/", params: params)
#form = page.forms[0]
#query = {"foo" => "foo_value", "bar" => "bar_value"}
#page = agent.post("http://example.com/", query: query)
#puts page.code
#puts page.body
#puts page.css("h1").first.inner_text
#puts page.title
# form = page.forms[0]
# query = {"foo" => "foo_value", "bar" => "bar_value"}
# page = agent.post("http://example.com/", query: query)
# puts page.code
# puts page.body
# puts page.css("h1").first.inner_text
# puts page.title

View File

@ -6,12 +6,11 @@ WebMock.stub(:get, "example.com/cookies2").to_return(headers: {"Set-Cookie" => "
WebMock.stub(:get, "example.com/cookies3").to_return(headers: {"Set-Cookie" => "id=456"})
WebMock.stub(:get, "example.com/secure_cookies").to_return(headers: {"Set-Cookie" => "id=123; Secure"})
WebMock.stub(:get, "example.com/paths").to_return(headers: {"Set-Cookie" => "id=123; Path=/paths"})
WebMock.stub(:get, "example.com/paths/hoge").to_return()
WebMock.stub(:get, "https://example.com/").to_return()
WebMock.stub(:get, "example.com/hoge/paths").to_return()
WebMock.stub(:get, "www.example.com").to_return()
WebMock.stub(:get, "example.com/meta_cookie").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/paths/hoge").to_return
WebMock.stub(:get, "https://example.com/").to_return
WebMock.stub(:get, "example.com/hoge/paths").to_return
WebMock.stub(:get, "www.example.com").to_return
WebMock.stub(:get, "example.com/meta_cookie").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/button").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/button").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/check_box").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/check_box").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>
@ -33,7 +32,7 @@ describe "Form Fields CheckBox" do
checkbox.checked?.should eq false
checkbox.check
checkbox.checked?.should eq true
# #click reverses check status
# #click reverses check status
checkbox.click
checkbox.checked?.should eq false
checkbox.click

View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/fields").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/fields").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/multi_select_list").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/multi_select_list").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/multi_select_list").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/multi_select_list").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

View File

@ -1,7 +1,6 @@
require "../spec_helper.cr"
WebMock.stub(:get, "example.com/form/radio_button").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/radio_button").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>
@ -26,7 +25,7 @@ describe "Form Fields RadioButton" do
radiobuttons.size.should eq 3
it "returns radiobutton check status" do
radiobuttons.map(&.checked?).should eq [false,false,false]
radiobuttons.map(&.checked?).should eq [false, false, false]
end
it "can change check status" do
@ -36,7 +35,7 @@ describe "Form Fields RadioButton" do
radiobutton.checked?.should eq true
radiobutton.uncheck
radiobutton.checked?.should eq false
# #click reverses check status
# #click reverses check status
radiobutton.click
radiobutton.checked?.should eq true
radiobutton.click

View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/select_list").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/select_list").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

View File

@ -1,7 +1,6 @@
require "../spec_helper"
WebMock.stub(:get, "example.com/form/textarea").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form/textarea").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

View File

@ -1,7 +1,6 @@
require "./spec_helper"
WebMock.stub(:get, "example.com/check_form").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/check_form").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>

View File

@ -1,8 +1,8 @@
require "./spec_helper"
WebMock.stub(:get, "http://example.com/?foo=bar&foo1=bar2")
WebMock.stub(:post, "http://example.com/post").
with(body: "email=foobar", headers: {"Content-Type" => "application/x-www-form-urlencoded"}).
to_return(body: "success")
WebMock.stub(:post, "http://example.com/post")
.with(body: "email=foobar", headers: {"Content-Type" => "application/x-www-form-urlencoded"})
.to_return(body: "success")
WebMock.stub(:get, "example.com/%E3%81%82%E3%81%82%E3%81%82")
describe "Mechanize HTTP test" do
@ -49,7 +49,7 @@ describe "Mechanize HTTP test" do
it "simple POST" do
agent = Mechanize.new
query = { "email" => "foobar" }
query = {"email" => "foobar"}
page = agent.post("http://example.com/post", query: query)
page.body.should eq "success"
page.code.should eq 200

View File

@ -35,14 +35,14 @@ describe "Mechanize Page test" do
it "can detect form by using form_with method, argument type: Hash" do
agent = Mechanize.new
page = agent.get("http://example.com/form")
form = page.form_with({"name" => "sample_form" })
form = page.form_with({"name" => "sample_form"})
form.name.should eq "sample_form"
end
it "can detect form by using form_with method, argument type: NamedTuple" do
agent = Mechanize.new
page = agent.get("http://example.com/form")
form = page.form_with({name: "sample_form" })
form = page.form_with({name: "sample_form"})
form.name.should eq "sample_form"
end
end

View File

@ -2,14 +2,12 @@ require "spec"
require "webmock"
require "../src/mechanize"
WebMock.stub(:get, "example.com")
WebMock.stub(:get, "fail_example.com").to_return(status: 500)
WebMock.stub(:get, "body_example.com").to_return(body: "hello")
WebMock.stub(:get, "another_domain.com/")
WebMock.stub(:get, "example.com/form").to_return(body:
<<-BODY
WebMock.stub(:get, "example.com/form").to_return(body: <<-BODY
<html>
<head>
<title>page_title</title>
@ -24,10 +22,10 @@ WebMock.stub(:get, "example.com/form").to_return(body:
</html>
BODY
)
WebMock.stub(:post, "example.com/post_path").
with(body: "name=foo&email=bar", headers: {"Content-Type" => "application/x-www-form-urlencoded"}).
to_return(body: "success")
WebMock.stub(:post, "example.com/post_path")
.with(body: "name=foo&email=bar", headers: {"Content-Type" => "application/x-www-form-urlencoded"})
.to_return(body: "success")
WebMock.stub(:post, "example.com/post_path").
with(body: "name=foo&email=bar&commit=submit", headers: {"Content-Type" => "application/x-www-form-urlencoded"}).
to_return(body: "success with button")
WebMock.stub(:post, "example.com/post_path")
.with(body: "name=foo&email=bar&commit=submit", headers: {"Content-Type" => "application/x-www-form-urlencoded"})
.to_return(body: "success with button")

View File

@ -10,27 +10,28 @@ class Mechanize
AGENT = {
"Mechanize" => "Mechanize/#{VERSION} Crystal/#{Crystal::VERSION} (https://github.com/Kanezoh/mechanize.cr)",
}
def initialize()
def initialize
@agent = MechanizeCr::HTTP::Agent.new
@agent.context = self
@agent.user_agent = AGENT["Mechanize"]
end
def get(uri : String | URI, headers = HTTP::Headers.new, params : Hash(String, String | Array(String)) = Hash(String,String).new)
def get(uri : String | URI, headers = HTTP::Headers.new, params : Hash(String, String | Array(String)) = Hash(String, String).new)
method = :get
page = @agent.fetch uri, method, headers, params
add_to_history(page)
#yield page if block_given?
# yield page if block_given?
page
end
def post(uri : String | URI, headers = HTTP::Headers.new, query : Hash(String, String | Array(String)) = Hash(String,String).new)
def post(uri : String | URI, headers = HTTP::Headers.new, query : Hash(String, String | Array(String)) = Hash(String, String).new)
node = Node.new
node["method"] = "POST"
node["enctype"] = "application/x-www-form-urlencoded"
form = MechanizeCr::Form.new(node)
query.each do |k,v|
query.each do |k, v|
node = Node.new
node["name"] = k
form.fields << MechanizeCr::FormContent::Field.new(node, v)
@ -44,13 +45,13 @@ class Mechanize
request_data = form.request_data
content_headers = ::HTTP::Headers{
"Content-Type" => form.enctype,
"Content-Length" => request_data.size.to_s,
"Content-Type" => form.enctype,
"Content-Length" => request_data.size.to_s,
}
headers.merge!(content_headers)
# fetch the page
page = @agent.fetch(uri, :post, headers: headers, params: {"value" => request_data }, referer: cur_page)
page = @agent.fetch(uri, :post, headers: headers, params: {"value" => request_data}, referer: cur_page)
headers.delete("Content-Type")
headers.delete("Content-Length")
add_to_history(page)
@ -81,7 +82,7 @@ class Mechanize
@agent.history.pop
end
def submit(form, button=nil)
def submit(form, button = nil)
form.add_button_to_query(button) if button
case form.method.upcase
when "POST"
@ -102,8 +103,8 @@ class Mechanize
history.push(page)
end
# Get maximum number of items allowed in the history.
# The default setting is 100 pages.
# Get maximum number of items allowed in the history.
# The default setting is 100 pages.
def max_history
history.max_size
end

View File

@ -6,11 +6,12 @@
# this property is used to send cookies to same origin resource.
class ::HTTP::Cookie
property origin : String?
def initialize(name : String, value : String, @path : String? = nil,
@expires : Time? = nil, @domain : String? = nil,
@secure : Bool = false, @http_only : Bool = false,
@samesite : SameSite? = nil, @extension : String? = nil,
@origin : String? = nil)
@expires : Time? = nil, @domain : String? = nil,
@secure : Bool = false, @http_only : Bool = false,
@samesite : SameSite? = nil, @extension : String? = nil,
@origin : String? = nil)
validate_name(name)
@name = name
validate_value(value)
@ -30,7 +31,7 @@ class ::HTTP::Cookie
end
if domain
host.try &.=~(/.*#{domain.try &.gsub(".", "\.")}$/)
host.try &.=~(/.*#{domain.try &.gsub(".", ".")}$/)
else
origin == host
end

View File

@ -1,13 +1,13 @@
require "./base_error"
class MechanizeCr::ElementNotFoundError < MechanizeCr::Error
getter element : Symbol
getter conditions : String
def initialize(element, conditions)
@element = element
@element = element
@conditions = conditions
super "Element #{element} with conditions #{conditions} was not found"
end
end

View File

@ -1,9 +1,11 @@
require "http/client"
class MechanizeCr::File
#property :body, :filename
# property :body, :filename
property :body, :code, uri, :response
def initialize(uri : URI, response : ::HTTP::Client::Response, body : String , code : Int32)
@uri = uri
def initialize(uri : URI, response : ::HTTP::Client::Response, body : String, code : Int32)
@uri = uri
@body = body
@code = code
end

View File

@ -11,35 +11,35 @@ require "./utils/element_matcher"
class MechanizeCr::Form
include MechanzeCr::ElementMatcher
getter node : Node | Lexbor::Node
getter fields : Array(FormContent::Field)
getter checkboxes : Array(FormContent::CheckBox)
getter node : Node | Lexbor::Node
getter fields : Array(FormContent::Field)
getter checkboxes : Array(FormContent::CheckBox)
getter radiobuttons : Array(FormContent::RadioButton)
getter selectboxes : Array(FormContent::MultiSelectList)
getter buttons : Array(FormContent::Button)
getter enctype : String
getter method : String
getter name : String
getter page : Page?
property action : String
getter selectboxes : Array(FormContent::MultiSelectList)
getter buttons : Array(FormContent::Button)
getter enctype : String
getter method : String
getter name : String
getter page : Page?
property action : String
def initialize(node : Node | Lexbor::Node, page : Page? = nil)
@enctype = node.fetch("enctype", "application/x-www-form-urlencoded")
@node = node
@fields = Array(FormContent::Field).new
@checkboxes = Array(FormContent::CheckBox).new
@radiobuttons = Array(FormContent::RadioButton).new
@selectboxes = Array(FormContent::MultiSelectList).new
@buttons = Array(FormContent::Button).new
@action = node.fetch("action", "")
@method = node.fetch("method", "GET").upcase
@name = node.fetch("name", "")
@clicked_buttons = Array(FormContent::Button).new
@page = page
#@mech = mech
@enctype = node.fetch("enctype", "application/x-www-form-urlencoded")
@node = node
@fields = Array(FormContent::Field).new
@checkboxes = Array(FormContent::CheckBox).new
@radiobuttons = Array(FormContent::RadioButton).new
@selectboxes = Array(FormContent::MultiSelectList).new
@buttons = Array(FormContent::Button).new
@action = node.fetch("action", "")
@method = node.fetch("method", "GET").upcase
@name = node.fetch("name", "")
@clicked_buttons = Array(FormContent::Button).new
@page = page
# @mech = mech
#@encoding = node['accept-charset'] || (page && page.encoding) || nil
#@ignore_encoding_error = false
# @encoding = node['accept-charset'] || (page && page.encoding) || nil
# @ignore_encoding_error = false
parse
end
@ -75,7 +75,7 @@ class MechanizeCr::Form
buttons << FormContent::Button.new(html_node, @node)
when "submit"
buttons << FormContent::SubmitButton.new(html_node, @node)
when"reset"
when "reset"
buttons << FormContent::ResetButton.new(html_node, @node)
when "image"
buttons << FormContent::ImageButton.new(html_node, @node)
@ -118,7 +118,7 @@ class MechanizeCr::Form
private def build_query_string(params : Array(Array(String)))
params.reduce("") do |acc, arr|
hash = { arr[0] => arr[1] }
hash = {arr[0] => arr[1]}
acc + URI::Params.encode(hash) + '&'
end.rchop
end
@ -144,9 +144,9 @@ class MechanizeCr::Form
radio_groups.each_value do |g|
checked = g.select(&.checked)
if checked.uniq.size > 1
#values = checked.map(&.value).join(', ').inspect
#name = checked.first.name.inspect
#raise Mechanize::Error,
# values = checked.map(&.value).join(', ').inspect
# name = checked.first.name.inspect
# raise Mechanize::Error,
# "radiobuttons #{values} are checked in the #{name} group, " \
# "only one is allowed"
raise MechanizeCr::Error.new
@ -182,8 +182,8 @@ class MechanizeCr::Form
def add_button_to_query(button)
unless button.form_node == @node
message = ""
"#{button.inspect} does not belong to the same page as " \
"the form #{@name.inspect} in #{@page.try &.uri}"
"#{button.inspect} does not belong to the same page as " \
"the form #{@name.inspect} in #{@page.try &.uri}"
message = "not a valid button"
raise ArgumentError.new(message)
end

View File

@ -1,10 +1,12 @@
class MechanizeCr::FormContent::Button < MechanizeCr::FormContent::Field
getter form_node : Node | Lexbor::Node
def initialize(node : Node | Lexbor::Node, form_node : Node | Lexbor::Node, value=nil)
def initialize(node : Node | Lexbor::Node, form_node : Node | Lexbor::Node, value = nil)
@form_node = form_node
super(node, value)
end
end
require "./reset_button"
require "./submit_button"
require "./image_button"

View File

@ -2,6 +2,7 @@ class MechanizeCr::FormContent::CheckBox < MechanizeCr::FormContent::RadioButton
def check
@checked = true
end
def query_value
[@name, @value || "on"]
end

View File

@ -1,15 +1,15 @@
class MechanizeCr::FormContent::Field
property value : String?
getter name : String
getter type : String
property value : String?
getter name : String
getter type : String
getter raw_value : String?
getter node : Node | Lexbor::Node
getter node : Node | Lexbor::Node
def initialize(node : Node | Lexbor::Node, value=nil)
@node = node
@name = node.fetch("name", "")
@value = value || node.fetch("value", nil)
@type = node.fetch("type", "")
def initialize(node : Node | Lexbor::Node, value = nil)
@node = node
@name = node.fetch("name", "")
@value = value || node.fetch("value", nil)
@type = node.fetch("type", "")
@raw_value = @value
end
@ -30,7 +30,7 @@ class MechanizeCr::FormContent::Field
def inspect # :nodoc:
"[%s:0x%x type: %s name: %s value: %s]" % [
self.class.name.sub(/MechanizeCr::FormContent::/, "").downcase,
object_id, type, name, value
object_id, type, name, value,
]
end
end

View File

@ -1,17 +1,17 @@
require "./option"
class MechanizeCr::FormContent::MultiSelectList
getter node : Lexbor::Node
getter name : String
getter type : String
property values : Array(String)
property options : Array(FormContent::Option)
getter node : Lexbor::Node
getter name : String
getter type : String
property values : Array(String)
property options : Array(FormContent::Option)
def initialize(node : Lexbor::Node)
@node = node
@name = node.fetch("name", "")
@type = node.fetch("type", "")
@values = Array(String).new
@node = node
@name = node.fetch("name", "")
@type = node.fetch("type", "")
@values = Array(String).new
@options = Array(FormContent::Option).new
node.css("option").each { |n|
@options << FormContent::Option.new(n, self)
@ -55,7 +55,7 @@ class MechanizeCr::FormContent::MultiSelectList
def inspect # :nodoc:
"[%s:0x%x type: %s name: %s values: [%s]]" % [
self.class.name.sub(/MechanizeCr::FormContent::/, "").downcase,
object_id, type, name, values.join(',')
object_id, type, name, values.join(','),
]
end
end

View File

@ -1,15 +1,15 @@
class MechanizeCr::FormContent::Option
getter select_list : FormContent::MultiSelectList
getter node : Lexbor::Node
getter text : String
getter value : String
getter selected : Bool
getter node : Lexbor::Node
getter text : String
getter value : String
getter selected : Bool
def initialize(node, select_list)
@node = node
@text = node.inner_text
@value = (node["value"] || node.inner_text)
@selected = node.has_key?("selected")
@node = node
@text = node.inner_text
@value = (node["value"] || node.inner_text)
@selected = node.has_key?("selected")
@select_list = select_list # The select list this option belongs to
end

View File

@ -3,7 +3,7 @@ class MechanizeCr::FormContent::RadioButton < MechanizeCr::FormContent::Field
def initialize(node : Node | Lexbor::Node, form : Form)
@checked = !!node.fetch("checked", nil)
@form = form
@form = form
super(node)
end
@ -24,33 +24,32 @@ class MechanizeCr::FormContent::RadioButton < MechanizeCr::FormContent::Field
checked
end
#def hash # :nodoc:
# def hash # :nodoc:
# @form.hash ^ @name.hash ^ @value.hash
#end
#
#def label
# end
#
# def label
# (id = self['id']) && @form.page.labels_hash[id] || nil
#end
#
#def text
# end
#
# def text
# label.text rescue nil
#end
#
#def [](key)
# end
#
# def [](key)
# @node[key]
#end
# end
# alias checked? checked
#def == other # :nodoc:
# def == other # :nodoc:
# self.class === other and
# other.form == @form and
# other.name == @name and
# other.value == @value
#end
#
#alias eql? == # :nodoc:
# end
#
# alias eql? == # :nodoc:
private def uncheck_peers
form.radiobuttons_with(name).try &.each do |b|

View File

@ -2,6 +2,7 @@ require "./page"
class MechanizeCr::History < Array(MechanizeCr::Page)
property max_size : Int32
def initialize(max_size = 100)
@max_size = max_size
super

View File

@ -19,7 +19,7 @@ module MechanizeCr
@user_agent = ""
end
def fetch(uri, method = :get, headers = HTTP::Headers.new, params = Hash(String,String).new,
def fetch(uri, method = :get, headers = HTTP::Headers.new, params = Hash(String, String).new,
referer = (current_page unless history.empty?))
uri = resolve_url(uri, referer)
set_request_headers(uri, headers)
@ -52,7 +52,7 @@ module MechanizeCr
def http_request(uri, method, params)
case uri.scheme.not_nil!.downcase
when "http", "https" then
when "http", "https"
case method
when :get
::HTTP::Client.get(uri, headers: request_headers)
@ -70,7 +70,7 @@ module MechanizeCr
@history.pop
end
# Get maximum number of items allowed in the history. The default setting is 100 pages.
# Get maximum number of items allowed in the history. The default setting is 100 pages.
def max_history
@history.max_size
end
@ -82,7 +82,7 @@ module MechanizeCr
private def set_request_headers(uri, headers)
reset_request_header_cookies
headers.each do |k,v|
headers.each do |k, v|
request_headers[k] = v
end
valid_cookies(uri).add_request_headers(request_headers)
@ -117,8 +117,8 @@ module MechanizeCr
private def save_response_cookies(response, uri, page)
if page.body =~ /Set-Cookie/
page.css("head meta[http-equiv=\"Set-Cookie\"]").each do |meta|
cookie = meta["content"].split(";")[0]
key,value = cookie.split("=")
cookie = meta["content"].split(";")[0]
key, value = cookie.split("=")
cookie = ::HTTP::Cookie.new(name: key, value: value)
save_cookies(uri, [cookie])
end
@ -148,7 +148,7 @@ module MechanizeCr
end
# fill host if host isn't set
if target_url.host.nil? && referer && referer_uri.try &.host
if target_url.host.nil? && referer && referer_uri.try &.host
target_url.host = referer_uri.not_nil!.host
end
# fill scheme if scheme isn't set

View File

@ -1,9 +1,9 @@
require "lexbor"
# This is a fake node used when sending post request.
class Node < Hash(String,String)
class Node < Hash(String, String)
def css(str)
[] of Hash(String,String)
[] of Hash(String, String)
end
def inner_text

View File

@ -1,5 +1,5 @@
module MechanzeCr::ElementMatcher
macro elements_with(singular, plural="")
macro elements_with(singular, plural = "")
{% plural = "#{singular.id}s" if plural.empty? %}
def {{plural.id}}_with(criteria)
{{plural.id}}_with(criteria){}