From 96802dbcafc1df25a242fc100d7f06a580960d86 Mon Sep 17 00:00:00 2001
From: Alex Reisner <alex@alexreisner.com>
Date: Tue, 6 Nov 2012 12:51:51 -0500
Subject: [PATCH] Upgrade Yahoo from old/current API to Yahoo BOSS.
---
README.md | 18 +--
lib/geocoder/configuration.rb | 2 +-
lib/geocoder/lookups/yahoo.rb | 57 +++-------
lib/oauth_util.rb | 106 ++++++++++++++++++
.../fixtures/yahoo_madison_square_garden.json | 96 ++++++++--------
test/fixtures/yahoo_no_results.json | 14 +--
.../yahoo_v1_madison_square_garden.json | 46 --------
test/fixtures/yahoo_v1_no_results.json | 10 --
test/lookup_test.rb | 6 -
test/services_test.rb | 39 +++----
test/test_helper.rb | 6 +-
11 files changed, 205 insertions(+), 195 deletions(-)
create mode 100644 lib/oauth_util.rb
delete mode 100644 test/fixtures/yahoo_v1_madison_square_garden.json
delete mode 100644 test/fixtures/yahoo_v1_no_results.json
diff --git a/README.md b/README.md
index 7e7b60c6..c1113188 100644
--- a/README.md
+++ b/README.md
@@ -263,7 +263,7 @@ By default Geocoder uses Google's geocoding API to fetch coordinates and street
Geocoder.configure do |config|
# geocoding service (see below for supported options):
- config.lookup = :yahoo
+ config.lookup = :yandex
# to use an API key:
config.api_key = "..."
@@ -304,17 +304,19 @@ The following is a comparison of the supported geocoding APIs. The "Limitations"
* **Limitations**: "You must not use or display the Content without a corresponding Google map, unless you are explicitly permitted to do so in the Maps APIs Documentation, or through written permission from Google." "You must not pre-fetch, cache, or store any Content, except that you may store: (i) limited amounts of Content for the purpose of improving the performance of your Maps API Implementation..."
* **Notes**: To use Google Premier set `Geocoder::Configuration.lookup = :google_premier` and `Geocoder::Configuration.api_key = [key, client, channel]`.
-#### Yahoo (`:yahoo`)
+#### Yahoo BOSS (`:yahoo`)
-* **API key**: optional in development (required for production apps)
-* **Key signup**: https://developer.apps.yahoo.com/wsregapp
-* **Quota**: 50,000 requests/day, more available by special arrangement
+Yahoo BOSS is **not a free service**. As of November 17, 2012 Yahoo no longer offers a free geocoding API.
+
+* **API key**: requires OAuth consumer key and secret (set `Geocoder::Configuration.api_key = [key, secret]`)
+* **Key signup**: http://developer.yahoo.com/boss/geo/
+* **Quota**: unlimited, but subject to usage fees
* **Region**: world
* **SSL support**: no
* **Languages**: ?
-* **Documentation**: http://developer.yahoo.com/geo/placefinder/guide/responses.html
-* **Terms of Service**: http://info.yahoo.com/legal/us/yahoo/maps/mapsapi/mapsapi-2141.html
-* **Limitations**: "YOU SHALL NOT... (viii) store or allow end users to store map imagery, map data or geocoded location information from the Yahoo! Maps APIs for any future use; (ix) use the stand-alone geocoder for any use other than displaying Yahoo! Maps or displaying points on Yahoo! Maps;"
+* **Documentation**: http://developer.yahoo.com/boss/geo/docs/index.html
+* **Terms of Service**: http://info.yahoo.com/legal/us/yahoo/boss/tou/?pir=ucJPcJ1ibUn.h.d.lVmlcbcEkoHjwJ_PvxG9SLK9VIbIQAw1XFrnDqY-
+* **Limitations**: No mass downloads, no commercial map production based on the data, no storage of data except for caching.
#### Bing (`:bing`)
diff --git a/lib/geocoder/configuration.rb b/lib/geocoder/configuration.rb
index c2df6d3e..dd426e0b 100644
--- a/lib/geocoder/configuration.rb
+++ b/lib/geocoder/configuration.rb
@@ -23,7 +23,7 @@ module Geocoder
#
# Geocoder.configure do |config|
# config.timeout = 5
- # config.lookup = :yahoo
+ # config.lookup = :yandex
# config.api_key = "2a9fsa983jaslfj982fjasd"
# config.units = :km
# end
diff --git a/lib/geocoder/lookups/yahoo.rb b/lib/geocoder/lookups/yahoo.rb
index f27beca0..adbe2e98 100644
--- a/lib/geocoder/lookups/yahoo.rb
+++ b/lib/geocoder/lookups/yahoo.rb
@@ -1,5 +1,6 @@
require 'geocoder/lookups/base'
require "geocoder/results/yahoo"
+require 'oauth_util'
module Geocoder::Lookup
class Yahoo < Base
@@ -12,47 +13,16 @@ module Geocoder::Lookup
def results(query)
return [] unless doc = fetch_data(query)
- doc = doc['ResultSet']
- if api_version(doc).to_i == 1 and r = version_1_results(doc)
- return r
- elsif api_version(doc).to_i == 2 and r = version_2_results(doc)
- return r
- else
- warn "Yahoo Geocoding API error: #{doc['Error']} (#{doc['ErrorMessage']})."
- return []
- end
- end
-
- def api_version(doc)
- if doc.include?('version')
- return doc['version'].to_f
- elsif doc.include?('@version')
- return doc['@version'].to_f
- end
- end
-
- def version_1_results(doc)
- if doc['Error'] == 0
- if doc['Found'] > 0
- return doc['Results']
- else
- return []
- end
- end
- end
-
- ##
- # Return array of results, or nil if an error.
- #
- def version_2_results(doc)
- # seems to have Error == 7 when no results, though this is not documented
- if [0, 7].include?(doc['Error'].to_i)
- if doc['Found'].to_i > 0
- r = doc['Result']
- return r.is_a?(Array) ? r : [r]
+ doc = doc['bossresponse']
+ if doc['responsecode'].to_i == 200
+ if doc['placefinder']['count'].to_i > 0
+ return doc['placefinder']['results']
else
return []
end
+ else
+ warn "Yahoo Geocoding API error: #{doc['responsecode']}."
+ return []
end
end
@@ -60,14 +30,17 @@ module Geocoder::Lookup
super.merge(
:location => query.sanitized_text,
:flags => "JXTSR",
- :gflags => "AC#{'R' if query.reverse_geocode?}",
- :locale => "#{Geocoder::Configuration.language}_US",
- :appid => Geocoder::Configuration.api_key
+ :gflags => "AC#{'R' if query.reverse_geocode?}"
)
end
def query_url(query)
- "http://where.yahooapis.com/geocode?" + url_query_string(query)
+ base_url = "http://yboss.yahooapis.com/geo/placefinder?"
+ parsed_url = URI.parse(base_url + url_query_string(query))
+ o = OauthUtil.new
+ o.consumer_key = Geocoder::Configuration.api_key[0]
+ o.consumer_secret = Geocoder::Configuration.api_key[1]
+ base_url + o.sign(parsed_url).query_string
end
end
end
diff --git a/lib/oauth_util.rb b/lib/oauth_util.rb
new file mode 100644
index 00000000..a98cb686
--- /dev/null
+++ b/lib/oauth_util.rb
@@ -0,0 +1,106 @@
+# A utility for signing an url using OAuth in a way that's convenient for debugging
+# Note: the standard Ruby OAuth lib is here http://github.com/mojodna/oauth
+# Source: http://gist.github.com/383159
+# License: http://gist.github.com/375593
+# Usage: see example.rb below
+
+require 'uri'
+require 'cgi'
+require 'openssl'
+require 'base64'
+
+class OauthUtil
+
+ attr_accessor :consumer_key, :consumer_secret, :token, :token_secret, :req_method,
+ :sig_method, :oauth_version, :callback_url, :params, :req_url, :base_str
+
+ def initialize
+ @consumer_key = ''
+ @consumer_secret = ''
+ @token = ''
+ @token_secret = ''
+ @req_method = 'GET'
+ @sig_method = 'HMAC-SHA1'
+ @oauth_version = '1.0'
+ @callback_url = ''
+ end
+
+ # openssl::random_bytes returns non-word chars, which need to be removed. using alt method to get length
+ # ref http://snippets.dzone.com/posts/show/491
+ def nonce
+ Array.new( 5 ) { rand(256) }.pack('C*').unpack('H*').first
+ end
+
+ def percent_encode( string )
+
+ # ref http://snippets.dzone.com/posts/show/1260
+ return URI.escape( string, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]") ).gsub('*', '%2A')
+ end
+
+ # @ref http://oauth.net/core/1.0/#rfc.section.9.2
+ def signature
+ key = percent_encode( @consumer_secret ) + '&' + percent_encode( @token_secret )
+
+ # ref: http://blog.nathanielbibler.com/post/63031273/openssl-hmac-vs-ruby-hmac-benchmarks
+ digest = OpenSSL::Digest::Digest.new( 'sha1' )
+ hmac = OpenSSL::HMAC.digest( digest, key, @base_str )
+
+ # ref http://groups.google.com/group/oauth-ruby/browse_thread/thread/9110ed8c8f3cae81
+ Base64.encode64( hmac ).chomp.gsub( /\n/, '' )
+ end
+
+ # sort (very important as it affects the signature), concat, and percent encode
+ # @ref http://oauth.net/core/1.0/#rfc.section.9.1.1
+ # @ref http://oauth.net/core/1.0/#9.2.1
+ # @ref http://oauth.net/core/1.0/#rfc.section.A.5.1
+ def query_string
+ pairs = []
+ @params.sort.each { | key, val |
+ pairs.push( "#{ percent_encode( key ) }=#{ percent_encode( val.to_s ) }" )
+ }
+ pairs.join '&'
+ end
+
+ # organize params & create signature
+ def sign( parsed_url )
+
+ @params = {
+ 'oauth_consumer_key' => @consumer_key,
+ 'oauth_nonce' => nonce,
+ 'oauth_signature_method' => @sig_method,
+ 'oauth_timestamp' => Time.now.to_i.to_s,
+ 'oauth_version' => @oauth_version
+ }
+
+ # if url has query, merge key/values into params obj overwriting defaults
+ if parsed_url.query
+ #@params.merge! CGI.parse( parsed_url.query )
+ CGI.parse( parsed_url.query ).each do |k,v|
+ if v.is_a?(Array) && v.count == 1
+ @params[k] = v.first
+ else
+ @params[k] = v
+ end
+ end
+ end
+
+ # @ref http://oauth.net/core/1.0/#rfc.section.9.1.2
+ @req_url = parsed_url.scheme + '://' + parsed_url.host + parsed_url.path
+
+ # create base str. make it an object attr for ez debugging
+ # ref http://oauth.net/core/1.0/#anchor14
+ @base_str = [
+ @req_method,
+ percent_encode( req_url ),
+
+ # normalization is just x-www-form-urlencoded
+ percent_encode( query_string )
+
+ ].join( '&' )
+
+ # add signature
+ @params[ 'oauth_signature' ] = signature
+
+ return self
+ end
+end
diff --git a/test/fixtures/yahoo_madison_square_garden.json b/test/fixtures/yahoo_madison_square_garden.json
index f649afe2..24161f60 100644
--- a/test/fixtures/yahoo_madison_square_garden.json
+++ b/test/fixtures/yahoo_madison_square_garden.json
@@ -1,52 +1,52 @@
{
- "@lang": "en-US",
- "ResultSet": {
- "@version": "2.0",
- "@lang": "en-US",
- "Error": "0",
- "ErrorMessage": "No error",
- "Locale": "en-US",
- "Found": "1",
- "Quality": "90",
- "Result": {
- "quality": "90",
- "latitude": "40.750381",
- "longitude": "-73.993988",
- "offsetlat": "40.750381",
- "offsetlon": "-73.993988",
- "radius": "100",
- "boundingbox": {
- "north": "40.750832",
- "south": "40.749931",
- "east": "-73.993393",
- "west": "-73.994591"
- },
- "name": "Madison Square Garden",
- "line1": "Madison Square Garden",
- "line2": "New York, NY 10001",
- "line3": "",
- "line4": "United States",
- "cross": "",
- "house": "",
- "street": "",
- "xstreet": "",
- "unittype": "",
- "unit": "",
- "postal": "10001",
- "neighborhood": "Garment District|Midtown|Midtown West|Manhattan",
- "city": "New York",
- "county": "New York County",
- "state": "New York",
- "country": "United States",
- "countrycode": "US",
- "statecode": "NY",
- "countycode": "",
- "timezone": "America/New_York",
- "areacode": "212",
- "uzip": "10001",
- "hash": "",
- "woeid": "23617041",
- "woetype": "20"
+ "bossresponse": {
+ "responsecode": "200",
+ "placefinder": {
+ "start": "0",
+ "count": "1",
+ "request": "flags=JXTSR&location=Madison%20Square%20Garden%2C%20NY%2C%20NY&%unsafe%appid=%5B%22dj0yJmk9ZmZ5NXFrNGhNcEthJmQ9WVdrOVFUSlhPV2x1TjJVbWNHbzlORE0wT0RFME9UWXkmcz1jb25zdW1lcnNlY3JldCZ4PTAy%22%2C%20%22b57b1b98eb21f171231f5b441cba505261d6c9bb%22%5D&gflags=AC&locale=en_US",
+ "results": [
+ {
+ "quality": "90",
+ "latitude": "40.750381",
+ "longitude": "-73.993988",
+ "offsetlat": "40.750381",
+ "offsetlon": "-73.993988",
+ "radius": "400",
+ "boundingbox": {
+ "north": "40.750832",
+ "south": "40.749931",
+ "east": "-73.993393",
+ "west": "-73.994591"
+ },
+ "name": "Madison Square Garden",
+ "line1": "Madison Square Garden",
+ "line2": "New York, NY 10001",
+ "line3": "",
+ "line4": "United States",
+ "cross": "",
+ "house": "",
+ "street": "",
+ "xstreet": "",
+ "unittype": "",
+ "unit": "",
+ "postal": "10001",
+ "neighborhood": "Garment District|Midtown|Midtown West|Manhattan",
+ "city": "New York",
+ "county": "New York County",
+ "state": "New York",
+ "country": "United States",
+ "countrycode": "US",
+ "statecode": "NY",
+ "countycode": "",
+ "timezone": "America/New_York",
+ "areacode": "212",
+ "uzip": "10001",
+ "hash": "",
+ "woeid": "23617041",
+ "woetype": "20"
+ }
+ ]
}
}
}
diff --git a/test/fixtures/yahoo_no_results.json b/test/fixtures/yahoo_no_results.json
index e97865df..0c65fddc 100644
--- a/test/fixtures/yahoo_no_results.json
+++ b/test/fixtures/yahoo_no_results.json
@@ -1,10 +1,10 @@
{
- "ResultSet":{
- "version":"1.0",
- "Error":0,
- "ErrorMessage":"No error",
- "Locale":"us_US",
- "Quality":10,
- "Found":0
+ "bossresponse": {
+ "responsecode": "200",
+ "placefinder": {
+ "start": "0",
+ "count": "0",
+ "request": "flags=JXTSR&location=asdfasdf28394782sdfj2983&%unsafe%appid=%5B%22dj0yJmk9ZmZ5NXFrNGhNcEthJmQ9WVdrOVFUSlhPV2x1TjJVbWNHbzlORE0wT0RFME9UWXkmcz1jb25zdW1lcnNlY3JldCZ4PTAy%22%2C%20%22b57b1b98eb21f171231f5b441cba505261d6c9bb%22%5D&gflags=AC&locale=en_US"
+ }
}
}
diff --git a/test/fixtures/yahoo_v1_madison_square_garden.json b/test/fixtures/yahoo_v1_madison_square_garden.json
deleted file mode 100644
index 6e54b8e7..00000000
--- a/test/fixtures/yahoo_v1_madison_square_garden.json
+++ /dev/null
@@ -1,46 +0,0 @@
-{
- "ResultSet":{
- "version":"1.0",
- "Error":0,
- "ErrorMessage":"No error",
- "Locale":"us_US",
- "Quality":90,
- "Found":1,
- "Results":[{
- "quality":90,
- "latitude":"40.750381",
- "longitude":"-73.993988",
- "offsetlat":"40.750381",
- "offsetlon":"-73.993988",
- "radius":100,
- "name":"Madison Square Garden",
- "line1":"Madison Square Garden",
- "line2":"New York, NY 10001",
- "line3":"",
- "line4":"United States",
- "house":"",
- "street":"",
- "xstreet":"",
- "unittype":"",
- "unit":"",
- "postal":"10001",
- "neighborhood":"",
- "city":"New York",
- "county":"New York County",
- "state":"New York",
- "country":"United States",
- "countrycode":"US",
- "statecode":"NY",
- "countycode":"",
- "uzip":"10001",
- "hash":"",
- "woeid":23617041,
- "woetype":20,
- "cross":"",
- "timezone":"America/New_York",
- "neighborhood":"Garment District|Midtown|Midtown West|Manhattan",
- "areacode":"212",
- "boundingbox":{"north":"40.750832","south":"40.749931","east":"-73.993393","west":"-73.994591"}
- }]
- }
-}
diff --git a/test/fixtures/yahoo_v1_no_results.json b/test/fixtures/yahoo_v1_no_results.json
deleted file mode 100644
index e97865df..00000000
--- a/test/fixtures/yahoo_v1_no_results.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "ResultSet":{
- "version":"1.0",
- "Error":0,
- "ErrorMessage":"No error",
- "Locale":"us_US",
- "Quality":10,
- "Found":0
- }
-}
diff --git a/test/lookup_test.rb b/test/lookup_test.rb
index 5b4550c6..a0c8cd22 100644
--- a/test/lookup_test.rb
+++ b/test/lookup_test.rb
@@ -29,12 +29,6 @@ class LookupTest < Test::Unit::TestCase
assert_match "key=MY_KEY", g.send(:query_url, Geocoder::Query.new("Madison Square Garden, New York, NY 10001, United States"))
end
- def test_yahoo_app_id
- Geocoder::Configuration.api_key = "MY_KEY"
- g = Geocoder::Lookup::Yahoo.new
- assert_match "appid=MY_KEY", g.send(:query_url, Geocoder::Query.new("Madison Square Garden, New York, NY 10001, United States"))
- end
-
def test_geocoder_ca_showpostal
Geocoder::Configuration.api_key = "MY_KEY"
g = Geocoder::Lookup::GeocoderCa.new
diff --git a/test/services_test.rb b/test/services_test.rb
index c689f369..9d204aac 100644
--- a/test/services_test.rb
+++ b/test/services_test.rb
@@ -8,10 +8,20 @@ class ServicesTest < Test::Unit::TestCase
Geocoder::Lookup.all_services_except_test.each do |l|
next if l == :google_premier # TODO: need to set keys to test
next if l == :freegeoip # does not use query string
+ # need to set API key for Yahoo for OAuth encoding
+ if l == :yahoo
+ Geocoder::Configuration.api_key = [
+ 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+ ]
+ else
+ Geocoder::Configuration.api_key = nil
+ end
url = Geocoder::Lookup.get(l).send(:query_url, Geocoder::Query.new(
"test", :params => {:one_in_the_hand => "two in the bush"}
))
- assert_match /one_in_the_hand=two\+in\+the\+bush/, url,
+ # should be "+"s for all lookups except Yahoo
+ assert_match /one_in_the_hand=two(%20|\+)in(%20|\+)the(%20|\+)bush/, url,
"Lookup #{l} does not appear to support arbitrary params in URL"
end
end
@@ -67,37 +77,20 @@ class ServicesTest < Test::Unit::TestCase
# --- Yahoo ---
- def test_yahoo_v1_no_results
- Geocoder::Configuration.lookup = :yahoo
- assert_equal [], Geocoder.search("no results v1")
- end
-
- def test_yahoo_v1_result_components
- Geocoder::Configuration.lookup = :yahoo
- result = Geocoder.search("madison square garden v1").first
- assert_equal "10001", result.postal_code
- end
-
- def test_yahoo_v1_address_formatting
- Geocoder::Configuration.lookup = :yahoo
- result = Geocoder.search("madison square garden v1").first
- assert_equal "Madison Square Garden, New York, NY 10001, United States", result.address
- end
-
- def test_yahoo_v2_no_results
+ def test_yahoo_no_results
Geocoder::Configuration.lookup = :yahoo
assert_equal [], Geocoder.search("no results")
end
- def test_yahoo_v2_result_components
+ def test_yahoo_result_components
Geocoder::Configuration.lookup = :yahoo
- result = Geocoder.search("madison square garden v2").first
+ result = Geocoder.search("madison square garden").first
assert_equal "10001", result.postal_code
end
- def test_yahoo_v2_address_formatting
+ def test_yahoo_address_formatting
Geocoder::Configuration.lookup = :yahoo
- result = Geocoder.search("madison square garden v2").first
+ result = Geocoder.search("madison square garden").first
assert_equal "Madison Square Garden, New York, NY 10001, United States", result.address
end
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 1f592ecf..7361b300 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -103,10 +103,8 @@ module Geocoder
raise TimeoutError if query.text == "timeout"
raise SocketError if query.text == "socket_error"
file = case query.text
- when "no results v1"; :v1_no_results
- when "madison square garden v1"; :v1_madison_square_garden
- when "no results"; :no_results
- else :madison_square_garden
+ when "no results"; :no_results
+ else :madison_square_garden
end
read_fixture "yahoo_#{file}.json"
end
--
GitLab