diff --git a/.rubocop.yml b/.rubocop.yml index 20fb34f..ce4f456 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -46,7 +46,7 @@ Metrics/CyclomaticComplexity: # There is a tradeoff between line length and line count. Metrics/ClassLength: - Max: 160 + Max: 170 # Keyword args are readable. Metrics/ParameterLists: diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ff82ee..518e81b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # Changelog + * [Unreleased] Add native Extract API support via `extract` and `extract_link` methods * [Unreleased] Add Image API support * [Unreleased] Add Markdown search and archive output support * [Unreleased] Raise `SerpApiError` for non-200 text responses, including 404 responses from `get('/invalid')` diff --git a/README.md b/README.md index 3fe4a9d..210d9e8 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ client.close - [Asynchronous searches](./demo/demo_async.rb) for submitting non-blocking jobs and retrieving completed results from the Search Archive API. - [Persistent connections and connection pooling](./demo/demo_thread_pool.rb) for reusing HTTP connections across searches. - Search results as Ruby hashes with `search`, token-efficient Markdown with `md`, or raw search-engine HTML with `html`. -- SDK methods for the [Image API](https://serpapi.com/image-api), [Location API](https://serpapi.com/locations-api), [Search Archive API](https://serpapi.com/search-archive-api), and [Account API](https://serpapi.com/account-api). +- SDK methods for the [Extract API](https://serpapi.com/extract-api), [Image API](https://serpapi.com/image-api), [Location API](https://serpapi.com/locations-api), [Search Archive API](https://serpapi.com/search-archive-api), and [Account API](https://serpapi.com/account-api). - Configurable HTTP timeouts and symbolized or string JSON keys. ## Response formats @@ -174,6 +174,46 @@ This code shows a simple solution to batch searches asynchronously into a [queue Here are some examples for some of our most popular APIs. You can find the full list of supported engines and parameters in our [documentation](https://serpapi.com/search-engine-apis). +### Extract API + +Turn any public web page into a fully structured JSON document: page information, meta tags, structured data (Open Graph, Twitter Card, JSON-LD, microdata), headings, inline links, images, media, lists, tables, forms, price offers, and normalized text content. + +```ruby +require 'serpapi' + +client = SerpApi::Client.new(api_key: ENV['SERPAPI_KEY']) +results = client.extract(url: 'https://example.com/') +pp results[:page_information] +``` + +The same method exposes two additional modes: + +- `map` performs fast same-origin URL discovery via page links, `robots.txt`, and `sitemap.xml`, and returns a `urls` array. +- `crawl` crawls pages on the same website up to `max_pages` / `max_depth` and returns a fully parsed JSON document for every visited page in a `pages` array. + +```ruby +# discover URLs of a website +results = client.extract(url: 'https://example.com/', mode: 'map') +pp results[:urls] + +# crawl a website and parse every visited page +results = client.extract(url: 'https://books.toscrape.com/', mode: 'crawl', max_pages: 3, max_depth: 1) +pp results[:pages] +``` + +Items carrying a navigable page URL (such as `inline_links`, `breadcrumbs`, `price_offers`, `urls`, and `pages`) are enriched with a `serpapi_extract_link`. Use `extract_link` to chain extractions without rebuilding the query: + +```ruby +results = client.extract(url: 'https://example.com/') +linked_page = client.extract_link(results[:inline_links].first[:serpapi_extract_link]) +``` + + * source code: [demo/demo_extract.rb](https://github.com/serpapi/serpapi-ruby/blob/master/demo/demo_extract.rb) + +> **Note:** URLs matching a website already covered by a dedicated SerpApi engine (such as Google, Bing, eBay, or Walmart search pages) are automatically redirected to that engine, and the response follows the matched engine's JSON structure. + +[See documentation](https://serpapi.com/extract-api) + ### Google Shopping Scrape Google Shopping results with product names, prices, ratings, and merchant information. diff --git a/demo/demo_extract.rb b/demo/demo_extract.rb new file mode 100644 index 0000000..22c35e4 --- /dev/null +++ b/demo/demo_extract.rb @@ -0,0 +1,62 @@ +# This script demonstrates the SerpApi Extract API (engine: extract). +# +# The Extract API turns any public web page into a fully structured JSON document. +# The same endpoint exposes three modes: +# - `scrape` (default): extract a single page into structured JSON +# (page information, meta tags, structured data, headings, inline links, text content, ...) +# - `map`: fast same-origin URL discovery via page links, robots.txt and sitemap.xml +# - `crawl`: crawl pages on the same website up to `max_pages` / `max_depth` +# and return a fully parsed JSON document for every visited page. +# +# Items carrying a navigable page URL (inline_links, breadcrumbs, price_offers, +# urls, pages, ...) are enriched with a `serpapi_extract_link` so extractions +# can be chained with `extract_link` without rebuilding the query. +# +# Ensure that the `SERPAPI_KEY` environment variable is set before running this script. +# +# doc: https://serpapi.com/extract-api + +raise 'SERPAPI_KEY environment variable must be set' if ENV['SERPAPI_KEY'].nil? + +require 'pp' +require 'serpapi' + +# client initialization, engine is set per call by the extract method +client = SerpApi::Client.new(api_key: ENV['SERPAPI_KEY']) + +# 1. scrape a single page (default mode) +puts '--- scrape: https://example.com/' +results = client.extract(url: 'https://example.com/') +pp results[:page_information] +pp results[:text_content] + +# 2. map: discover same-origin URLs of a website +puts '--- map: https://books.toscrape.com/' +results = client.extract(url: 'https://books.toscrape.com/', mode: 'map', max_pages: 10) +urls = results[:urls] || [] +puts "discovered #{urls.size} urls" +pp urls.first(5) + +# 3. crawl: parse every visited page up to max_pages / max_depth +puts '--- crawl: https://books.toscrape.com/' +results = client.extract(url: 'https://books.toscrape.com/', mode: 'crawl', max_pages: 3, max_depth: 1) +pages = results[:pages] || [] +pages.each do |page| + puts "#{page.dig(:page_information, :title)} - #{page.dig(:page_information, :url)}" +end + +# 4. chain extractions using serpapi_extract_link +puts '--- chain: follow the first inline link of https://example.com/' +results = client.extract(url: 'https://example.com/') +link = results[:inline_links]&.first&.dig(:serpapi_extract_link) +if link + linked_page = client.extract_link(link) + pp linked_page[:page_information] +else + puts 'no inline link found to follow' +end + +# close the underlying persistent connection +client.close +puts 'done' +exit 0 diff --git a/lib/serpapi.rb b/lib/serpapi.rb index 9125ca9..a4a3bf2 100644 --- a/lib/serpapi.rb +++ b/lib/serpapi.rb @@ -10,6 +10,9 @@ module SerpApi # load require 'json' +# parse serpapi_extract_link URLs +require 'uri' + # implementation require_relative 'serpapi/version' require_relative 'serpapi/error' diff --git a/lib/serpapi/client.rb b/lib/serpapi/client.rb index c872291..4d5cc46 100644 --- a/lib/serpapi/client.rb +++ b/lib/serpapi/client.rb @@ -136,6 +136,85 @@ def md(params = {}) get('/search', :md, force_output(params, 'md')) end + # Extract structured data from any public web page using the Extract API. + # + # The Extract API turns a URL into a fully structured JSON document: + # page information, meta tags, Open Graph / Twitter Card / Dublin Core, + # JSON-LD, microdata, breadcrumbs, headings, inline links / images / videos, + # scripts, stylesheets, feeds, lists, tables, forms, sections, price offers, + # and normalized text content. + # + # Modes (via the `mode` parameter): + # * `scrape` (default): extract a single page. + # * `map`: fast same-origin URL discovery, returns a `urls` array. + # * `crawl`: crawl the website up to `max_pages` / `max_depth`, returns a `pages` array. + # + # **Example:** + # + # ```ruby + # client = SerpApi::Client.new(api_key: ENV['SERPAPI_KEY']) + # + # # scrape a single page + # result = client.extract(url: 'https://example.com/') + # pp result[:page_information] + # + # # crawl a website + # result = client.extract(url: 'https://books.toscrape.com/', mode: 'crawl', max_pages: 3, max_depth: 1) + # pp result[:pages] + # ``` + # + # doc: https://serpapi.com/extract-api + # + # @param [Hash] params Extract API parameters, must include :url + # (target absolute http(s) URL). Optional parameters: + # mode, max_pages, max_depth, include_subdomains, + # include_sitemap, device, prefetch_delay, timeout, + # include_tags, exclude_tags, include_markdown, + # include_inline_assets. + # @return [Hash] extraction results formatted as a Hash. + def extract(params = {}) + raise SerpApiError, "params must be hash, not: #{params.class}" unless params.instance_of?(Hash) + raise SerpApiError, 'url parameter is required by the extract API' if params[:url].to_s.empty? && params['url'].to_s.empty? + + # also drop string-keyed engines; 'engine' => nil removes one inherited + # from the client defaults, since query compacts nil values + search(params.reject { |key, _| key.to_s == 'engine' }.merge('engine' => nil, engine: 'extract')) + end + + # Follow a `serpapi_extract_link` returned by the Extract API. + # + # Extract API results enrich items carrying a navigable page URL + # (inline_links, breadcrumbs, price_offers, urls, pages, ...) with a + # `serpapi_extract_link` so extractions can be chained without rebuilding + # the query. This method re-issues that link through the client, + # applying the client `api_key` and default parameters. An `api_key` + # embedded in the link is ignored. + # + # **Example:** + # + # ```ruby + # result = client.extract(url: 'https://example.com/') + # link = result[:inline_links].first[:serpapi_extract_link] + # linked_page = client.extract_link(link) + # ``` + # + # @param [String] link `serpapi_extract_link` URL from a previous extraction. + # @param [Hash] params optional parameter overrides. + # @return [Hash] extraction results formatted as a Hash. + def extract_link(link, params = {}) + raise SerpApiError, 'link must be a serpapi.com URL' unless link.to_s.match?(%r{\Ahttps://serpapi\.com/}i) + + link_params = URI.decode_www_form(URI.parse(link).query.to_s) + .to_h + .transform_keys(&:to_sym) + raise SerpApiError, "link does not target the extract engine: #{link}" if link_params[:engine] != 'extract' + + # never let a link (possibly from scraped content) choose the account + link_params = link_params.reject { |key, _| key == :api_key } + + extract(link_params.merge(params.transform_keys(&:to_sym))) + end + # Get location using Location API # # example: spec/serpapi/location_api_spec.rb diff --git a/spec/serpapi/client/example/example_extract_spec.rb b/spec/serpapi/client/example/example_extract_spec.rb new file mode 100644 index 0000000..6c6b387 --- /dev/null +++ b/spec/serpapi/client/example/example_extract_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +describe 'example: extract' do + it 'extracts structured data from a web page' do + # initialize the serp api client + client = SerpApi::Client.new(api_key: api_key) + # extract a single page using the serpapi extract engine + results = client.extract(url: 'https://example.com/') + expect(results[:page_information]).not_to be_nil, "No page information found! keys available: #{results.keys}" + + # print the output of the response in formatted JSON + # pp results[:page_information] + # doc: https://serpapi.com/extract-api + end + + it 'maps same-origin urls of a website' do + client = SerpApi::Client.new(api_key: api_key) + results = client.extract(url: 'https://example.com/', mode: 'map') + expect(results[:urls]).not_to be_nil, "No urls found! keys available: #{results.keys}" + end + + it 'crawls a website and returns parsed pages' do + client = SerpApi::Client.new(api_key: api_key) + results = client.extract(url: 'https://books.toscrape.com/', mode: 'crawl', max_pages: 2, max_depth: 1) + expect(results[:pages]).not_to be_nil, "No pages found! keys available: #{results.keys}" + end +end diff --git a/spec/serpapi/client/extract_api_spec.rb b/spec/serpapi/client/extract_api_spec.rb new file mode 100644 index 0000000..092d722 --- /dev/null +++ b/spec/serpapi/client/extract_api_spec.rb @@ -0,0 +1,117 @@ +require 'spec_helper' + +describe 'Extract API' do + let(:client) { SerpApi::Client.new(api_key: 'dummy_secret_key', persistent: false) } + + # Returns the query parameters sent over HTTP, after merging client defaults. + def sent_params + http = double + response = double(status: 200, body: '{}', headers: { 'Content-Type' => 'application/json' }) + sent = nil + allow(HTTP).to receive(:timeout).and_return(http) + allow(http).to receive(:get) do |_url, params:| + sent = params + response + end + yield + sent + end + + describe '#extract' do + it 'forces the extract engine and passes the url' do + expect(client).to receive(:search).with(hash_including(engine: 'extract', url: 'https://example.com/')) + client.extract(url: 'https://example.com/') + end + + it 'accepts extra parameters like mode, max_pages and max_depth' do + expect(client).to receive(:search).with( + hash_including( + engine: 'extract', + url: 'https://books.toscrape.com/', + mode: 'crawl', + max_pages: 3, + max_depth: 1 + ) + ) + client.extract(url: 'https://books.toscrape.com/', mode: 'crawl', max_pages: 3, max_depth: 1) + end + + it 'overrides a conflicting engine parameter' do + expect(client).to receive(:search).with(hash_including(engine: 'extract')) + client.extract(url: 'https://example.com/', engine: 'google') + end + + it 'replaces a string-keyed engine parameter' do + params = sent_params { client.extract('url' => 'https://example.com/', 'engine' => 'google') } + expect(params.select { |key, _| key.to_s == 'engine' }).to eq(engine: 'extract') + end + + it 'replaces a string-keyed engine from the client defaults' do + client = SerpApi::Client.new('engine' => 'google', api_key: 'dummy_secret_key', persistent: false) + params = sent_params { client.extract(url: 'https://example.com/') } + expect(params.select { |key, _| key.to_s == 'engine' }).to eq(engine: 'extract') + end + + it 'raises an error when url is missing' do + expect { client.extract }.to raise_error(SerpApi::SerpApiError, /url parameter is required/) + expect { client.extract(mode: 'map') }.to raise_error(SerpApi::SerpApiError, /url parameter is required/) + end + + it 'raises an error when params is not a hash' do + expect { client.extract('https://example.com/') }.to raise_error(SerpApi::SerpApiError, /params must be hash/) + end + end + + describe '#extract_link' do + let(:link) do + 'https://serpapi.com/search.json?device=desktop&engine=extract&url=https%3A%2F%2Fiana.org%2Fdomains%2Fexample' + end + + it 'parses the link query and re-issues the search through the client' do + expect(client).to receive(:search).with( + hash_including( + engine: 'extract', + device: 'desktop', + url: 'https://iana.org/domains/example' + ) + ) + client.extract_link(link) + end + + it 'applies parameter overrides' do + expect(client).to receive(:search).with(hash_including(device: 'mobile')) + client.extract_link(link, device: 'mobile') + end + + it 'ignores an api_key embedded in the link' do + params = sent_params { client.extract_link("#{link}&api_key=link_key") } + expect(params[:api_key]).to eq('dummy_secret_key') + expect(params.keys.map(&:to_s).count('api_key')).to eq(1) + end + + it 'replaces link parameters with string-keyed overrides' do + params = sent_params { client.extract_link(link, 'device' => 'mobile') } + expect(params.select { |key, _| key.to_s == 'device' }).to eq(device: 'mobile') + end + + it 'keeps the extract engine when overrides set another engine' do + params = sent_params { client.extract_link(link, engine: 'google') } + expect(params.select { |key, _| key.to_s == 'engine' }).to eq(engine: 'extract') + end + + it 'rejects links without a url' do + expect { client.extract_link('https://serpapi.com/search.json?engine=extract') } + .to raise_error(SerpApi::SerpApiError, /url parameter is required/) + end + + it 'rejects non serpapi.com links' do + expect { client.extract_link('https://evil.com/search.json?engine=extract') } + .to raise_error(SerpApi::SerpApiError, /must be a serpapi\.com URL/) + end + + it 'rejects links targeting another engine' do + expect { client.extract_link('https://serpapi.com/search.json?engine=google&q=coffee') } + .to raise_error(SerpApi::SerpApiError, /does not target the extract engine/) + end + end +end