Getting Started - michaeltelford/wgit GitHub Wiki

require 'wgit'

crawler = Wgit::Crawler.new # Uses Typhoeus -> libcurl underneath. It's fast!
url = Wgit::Url.new 'https://wikileaks.org/What-is-Wikileaks.html'

doc = crawler.crawl url # Or use #crawl_site(url) { |doc| ... } etc.
crawler.last_response.class # => Wgit::Response is a wrapper for Typhoeus::Response.

doc.class # => Wgit::Document
doc.class.public_instance_methods(false).sort # => [
#   :==, :[], :at_css, :at_xpath, :author, :author=, :base, :base=, :base_url, :content, :css,
#   :description, :description=, :empty?, :external_links, :extract, :html, :inspect,
#   :internal_absolute_links, :internal_links, :keywords, :keywords=, :links, :links=, :nearest_fragment,
#   :no_index?, :parser, :score, :search, :search!, :size, :stats, :text, :text=, :title, :title=, :to_h,
#   :to_json, :url, :xpath
# ]

doc.url   # => "https://wikileaks.org/What-is-Wikileaks.html"
doc.title # => "WikiLeaks - What is WikiLeaks"
doc.stats # => {
          #   :url=>44, :html=>28133, :title=>17, :keywords=>0,
          #   :links=>35, :text=>67, :text_bytes=>13735
          # }
doc.links # => ["#submit_help_contact", "#submit_help_tor", "#submit_help_tips", ...]
doc.text  # => ["The Courage Foundation is an international organisation that <snip>", ...]

results = doc.search 'corruption' # Searches doc.text for the given query.
results.first # => "ial materials involving war, spying and corruption.
              #     It has so far published more"

require 'wgit'

include Wgit::DSL

ENV['WGIT_CONNECTION_STRING'] = 'mongodb://user:password@localhost/crawler'

start 'https://www.bing.com/search?q=aid+workers'

extract :query, "//input[@id='sb_form_q']/@value"
extract(
  :results, "//li[@class='b_algo']",
  singleton: false, text_content_only: false
) do |results|
    # Map the results into a preferable format.
    results.map do |result|
      {
        title: result.at_xpath('./h2/a').content,
        url:   result.at_xpath('./div/div/cite').content,
        text:  result.at_xpath('./div/p').content,
      }
    end
end

index

require 'wgit'
require 'wgit/core_ext' # Provides the String#to_url and Enumerable#to_urls methods.

crawler = Wgit::Crawler.new
doc     = crawler.crawl 'https://www.facebook.com'.to_url

# Provide your own xpath (or css selector) to search the HTML using Nokogiri underneath.
href = doc.at_xpath "//link[@rel='stylesheet']/@href"

href.content # => "https://static.xx.fbcdn.net/rsrc.php/v3/y1/l/0,cross/NvZ4mNTW3Fd.css"
href.class   # => Nokogiri::XML::Attr

css = crawler.crawl href.content.to_url
css[0..50] # => "._3_s0._3_s0{border:0;display:flex;height:44px;min-"

require 'wgit'
require 'wgit/core_ext' # => Provides the String#to_url and Enumerable#to_urls methods.

my_pages_keywords = ['Everest', 'mountaineering school', 'adventure']
my_pages_missing_keywords = []

competitor_urls = [
  'http://altitudejunkies.com',
  'http://www.mountainmadness.com',
  'http://www.adventureconsultants.com'
].to_urls

crawler = Wgit::Crawler.new

crawler.crawl(*competitor_urls) do |doc|
  # If there are keywords present in the web document.
  if doc.keywords.respond_to? :-
    puts "The keywords for #{doc.url} are: \n#{doc.keywords}\n\n"
    my_pages_missing_keywords.concat(doc.keywords - my_pages_keywords)
  end
end

if my_pages_missing_keywords.empty?
  puts 'Your pages are missing no keywords, nice one!'
else
  puts 'Your pages compared to your competitors are missing the following keywords:'
  puts my_pages_missing_keywords.uniq
end

require 'wgit'

Wgit::Database.adapter_class = Wgit::Database::InMemory

# Use Wgit...

# By default Wgit will use the MongoDB class but we set it here explicitly for completeness.
Wgit::Database.adapter_class = Wgit::Database::MongoDB

db = Wgit::Database.new '<connection_string>'

# Mongo requires certain indexes to exist for searching, so we create them (once).
db.create_collections
db.create_unique_indexes
Wgit::Model.set_default_search_fields(db) # Creates a search index in Mongo using the Wgit::Model search fields.

# We set the search fields via the Wgit::Model passing the db param so Wgit's search methods abide by these fields too.
Wgit::Model.set_search_fields({ my_field: 2, ... }, db) # my_field has a search weight/priority of 2.
# OR
Wgit::Model.set_search_fields([:my_field, ...], db) # With a default weight of 1.

require 'wgit'

### CONNECT TO THE DATABASE ###

Wgit::Database.adapter_class = Wgit::Database::MongoDB

# In the absence of a connection string parameter, ENV['WGIT_CONNECTION_STRING'] will be used.
db = Database.new '<connection_string>'

### SEED SOME DATA ###

# Here we create our own document rather than crawling the web (which works in the same way).
# We provide the web page's URL and HTML Strings.
doc = Wgit::Document.new(
  'http://test-url.com',
  "<html><p>How now brown cow.</p><a href='http://www.google.co.uk'>Click me!</a></html>"
)
db.upsert doc

### SEARCH THE DATABASE ###

# Searching the database returns Wgit::Document's which have fields containing the query.
query = 'cow'
results = db.search query

# By default, the MongoDB ranking applies i.e. results.first has the most hits.
# Because results is an Array of Wgit::Document's, we can custom sort/rank e.g.
# `results.sort_by! { |doc| doc.url.crawl_duration }` ranks via page load times with
# results.first being the fastest. Any Wgit::Document attribute can be used, including
# those you define yourself by extending the API.

top_result = results.first
top_result.class           # => Wgit::Document
doc.url == top_result.url  # => true

### PULL OUT THE BITS THAT MATCHED OUR QUERY ###

# Searching each result gives the matching text snippets from that Wgit::Document.
top_result.search(query).first # => "How now brown cow."

### SEED URLS TO BE CRAWLED LATER ###

db.upsert top_result.external_links
urls_to_crawl = db.uncrawled_urls # => Results will include top_result.external_links.

require 'wgit'

# The default text_elements cover most visible page text but let's say we
# have a <table> element with text content that we want.
Wgit::HTMLToText.text_elements[:table] = :block # or :inline

doc = Wgit::Document.new(
  'http://some_url.com',
  <<~HTML
  <html>
    <p>Hello world!</p>
    <table>My table</table>
  </html>
  HTML
)

# Now every crawled Document#text will include <table> text content.
doc.text            # => ["Hello world!", "My table"]
doc.search('table') # => ["My table"]

require 'wgit'

# Let's get all the page's <table> elements.
Wgit::Document.define_extractor(
  :tables,                  # Wgit::Document#tables will return the page's tables.
  '//table',                # The xpath to extract the tables.
  singleton: false,         # True returns the first table found, false returns all.
  text_content_only: false, # True returns the table text, false returns the Nokogiri object.
) do |tables|
  # Here we can inspect/manipulate the tables before they're set as Wgit::Document#tables.
  tables
end

# Our Document has a table which we're interested in. Note it doesn't matter how the Document
# is initialised e.g. manually (as below) or via Wgit::Crawler methods etc.
doc = Wgit::Document.new(
  'http://some_url.com',
  <<~HTML
  <html>
    <p>Hello world! Welcome to my site.</p>
    <table>
      <tr><th>Name</th><th>Age</th></tr>
      <tr><td>Socrates</td><td>101</td></tr>
      <tr><td>Plato</td><td>106</td></tr>
    </table>
    <p>I hope you enjoyed your visit :-)</p>
  </html>
  HTML
)

# Call our newly defined method to obtain the table data we're interested in.
tables = doc.tables

# Both the collection and each table within the collection are plain Nokogiri objects.
tables.class       # => Nokogiri::XML::NodeSet
tables.first.class # => Nokogiri::XML::Element

# Note, the Document's stats now include our 'tables' extractor.
doc.stats # => {
#   :url=>19, :html=>242, :links=>0, :text=>8, :text_bytes=>91, :tables=>1
# }