From e90eb44cb7f688538d2bf31c25d0573f67d6fa2a Mon Sep 17 00:00:00 2001 From: Martin Tithonium Date: Thu, 9 Dec 2010 15:14:33 -0800 Subject: [PATCH] New UrlMetrics fields, and support for calculating 'missing' fields. Added constants for all of the new fields now returned by UrlMetrics. Added a list of fields whose values are not returned by UrlMetrics, but which can be calculated from other fields. For example, followed internal links can be calculated from total followed links minus followed external links. If the neccessary fields are present in the response, the calculated fields will automatically be added. If you request the calculated fields, the request will actually be for the fields that it depends upon. [Internal tracking: TUBO-869] --- README.rdoc | 109 +++++++--- Rakefile | 2 +- lib/linkscape/constants.rb | 147 ++++++++++++- lib/linkscape/constants/anchor-metrics.rb | 26 +-- lib/linkscape/constants/link-metrics.rb | 36 ++-- lib/linkscape/constants/url-metrics.rb | 242 ++++++++++++++++++---- lib/linkscape/response.rb | 13 +- linkscape.gemspec | 48 ++--- 8 files changed, 490 insertions(+), 133 deletions(-) diff --git a/README.rdoc b/README.rdoc index 71d1d0a..d4e373f 100644 --- a/README.rdoc +++ b/README.rdoc @@ -132,40 +132,98 @@ Depending on the type of data point return, you may access certain data points i === Source/Target/URL Metrics -* :status -* :fq_domain_mozrank -* :pl_domain_links -* :pl_domain_external_links +* :all_external_links +* :canonical_internal_id +* :canonical_url +* :cblocks_linking +* :domain_authority +* :domain_authority_raw +* :external_links +* :external_mozrank +* :external_mozrank_raw * :fq_domain +* :fq_domain_all_external_links +* :fq_domain_external_links +* :fq_domain_external_mozrank_sum +* :fq_domain_external_mozrank_sum_raw +* :fq_domain_fq_domains_linking +* :fq_domain_internal_links +* :fq_domain_juice_fq_domains_linking +* :fq_domain_juice_internal_links +* :fq_domain_juice_links +* :fq_domain_juice_pl_domains_linking +* :fq_domain_links +* :fq_domain_mozrank +* :fq_domain_mozrank_raw +* :fq_domain_mozrank_sum +* :fq_domain_mozrank_sum_raw * :fq_domain_moztrust +* :fq_domain_moztrust_raw * :fq_domain_pl_domains_linking -* :pl_domain -* :url +* :fq_domain_unfollowed_external_links +* :fq_domain_unfollowed_fq_domains_linking +* :fq_domain_unfollowed_internal_links +* :fq_domain_unfollowed_links +* :fq_domain_unfollowed_pl_domains_linking +* :fq_domain_updated_at +* :fq_domains_linking +* :internal_id +* :internal_links +* :ips_linking +* :juice_cblocks_linking +* :juice_fq_domains_linking +* :juice_internal_links +* :juice_ips_linking +* :juice_links +* :juice_pl_domains_linking +* :links +* :mozrank +* :mozrank_raw +* :moztrust +* :moztrust_raw +* :page_authority * :page_authority_raw +* :pl_domain +* :pl_domain_all_external_links +* :pl_domain_cblocks_linking +* :pl_domain_external_links +* :pl_domain_external_mozrank_sum * :pl_domain_external_mozrank_sum_raw -* :links -* :external_mozrank +* :pl_domain_internal_links +* :pl_domain_ips_linking +* :pl_domain_juice_cblocks_linking +* :pl_domain_juice_internal_links +* :pl_domain_juice_ips_linking +* :pl_domain_juice_links +* :pl_domain_juice_pl_domains_linking +* :pl_domain_links * :pl_domain_mozrank -* :juice_links -* :title -* :fq_domains_linking -* :page_authority -* :fq_domain_external_mozrank_sum_raw -* :pl_domain_moztrust -* :fq_domain_external_links -* :domain_authority_raw -* :canonical_url +* :pl_domain_mozrank_raw +* :pl_domain_mozrank_sum * :pl_domain_mozrank_sum_raw -* :fq_domain_links -* :all -* :mozrank +* :pl_domain_moztrust +* :pl_domain_moztrust_raw * :pl_domain_pl_domains_linking -* :external_links -* :fq_domain_fq_domains_linking +* :pl_domain_unfollowed_cblocks_linking +* :pl_domain_unfollowed_external_links +* :pl_domain_unfollowed_internal_links +* :pl_domain_unfollowed_ips_linking +* :pl_domain_unfollowed_links +* :pl_domain_unfollowed_pl_domains_linking +* :pl_domain_updated_at * :pl_domains_linking -* :domain_authority -* :fq_domain_mozrank_sum_raw -* :moztrust +* :status +* :title +* :unfollowed_cblocks_linking +* :unfollowed_external_links +* :unfollowed_fq_domains_linking +* :unfollowed_internal_links +* :unfollowed_ips_linking +* :unfollowed_links +* :unfollowed_pl_domains_linking +* :updated_at +* :url + === Link Metrics @@ -182,7 +240,6 @@ Depending on the type of data point return, you may access certain data points i * :text * :internal_subdomains_linking * :external_domains_linking -* :all * :record_id * :external_pages_linking diff --git a/Rakefile b/Rakefile index 0989577..fe011ee 100644 --- a/Rakefile +++ b/Rakefile @@ -9,7 +9,7 @@ begin gem.description = %Q{Provides an interface to SEOmoz's suite of APIs, including the free and site intelligence APIs.} gem.email = %q{api@seomoz.org} gem.homepage = "http://github.com/seomoz/linkscape-gem" - gem.authors = ["Marty Smyth", "Jeff Pollard"] + gem.authors = ["Martin Tithonium", "Jeff Pollard", "Bryce Howard"] gem.add_dependency "ruby-hmac", ">= 0" # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings end diff --git a/lib/linkscape/constants.rb b/lib/linkscape/constants.rb index aeb28bb..0f9584e 100644 --- a/lib/linkscape/constants.rb +++ b/lib/linkscape/constants.rb @@ -242,6 +242,98 @@ module Constants :desc => %Q[The pay-level domain (PL domain) as it's identified in the Linkscape index], }, + :ued => { + :key => :all_external_links, + :name => 'All external links page to page', + :desc => %Q[The number of external links from one page to another (included followed and nofollowed).] + }, + :ujfq => { + :key => :juice_fq_domains_linking, + :name => 'Followed Domains Linking Page', + :desc => %Q[The number of unique domains with followed links to the target url.] + }, + :ujp => { + :key => :juice_ips_linking, + :name => 'Followed IPs Linking', + :desc => %Q[The number unique IPs with a followable link to a target url.] + }, + :uip => { + :key => :ips_linking, + :name => 'IPs Linking', + :desc => %Q[The total number of unique IPs linking to a target url.] + }, + :ujpl => { + :key => :juice_pl_domains_linking, + :name => 'Followed Domains to Page', + :desc => %Q[The number of unique domains with followed links to a given url.] + }, + :uib => { + :key => :cblocks_linking, + :name => 'All Cblock Linking', + :desc => %Q[The total number unique cblocks linking to a page.] + }, + :ujb => { + :key => :juice_cblocks_linking, + :name => 'Followed CBLocks Linking', + :desc => %Q[The total number unique cblocks with followed links to a page.] + }, + :fjid => { + :key => :fq_domain_juice_links, + :name => 'Followed Subdomain Linking Domains', + :desc => %Q[A count of all unique subdomains with followed links to the target domain.] + }, + :fed => { + :key => :fq_domain_all_external_links, + :name => 'Subdomain External Links', + :desc => %Q[The total number (followed and nofollowed) external links to the subdomain of the url.] + }, + :fjf => { + :key => :fq_domain_juice_fq_domains_linking, + :name => 'Followed Subdomain Subdomains Links', + :desc => %Q[The number of subdomains with followed links to the subdomain of the url.] + }, + :fjd => { + :key => :fq_domain_juice_pl_domains_linking, + :name => 'Followed Domain Subdomains Links', + :desc => %Q[The number of unique domains with followed links to the subdomain of the url.] + }, + :pjid => { + :key => :pl_domain_juice_links, + :name => 'Followed Root Domain Links', + :desc => %Q[The total number of followed links (both internal and external) from a page to a domain.] + }, + :ped => { + :key => :pl_domain_all_external_links, + :name => 'All Root Domain External Links', + :desc => %Q[The total number of external links (both followed and no-followed) from a page to a domain.] + }, + :pjd => { + :key => :pl_domain_juice_pl_domains_linking, + :name => 'All Followed Root Domains Linking Domain', + :desc => %Q[The total number of followed root domains linking to the target's domain.] + }, + :pip => { + :key => :pl_domain_ips_linking, + :name => 'IPs Linking to Domain', + :desc => %Q[The total number of unique IPs linking to the target's domain.] + }, + :pjip => { + :key => :pl_domain_juice_ips_linking, + :name => 'Followed IPs Linking to Domain', + :desc => %Q[The total number of unique IPs with followed links to the target's domain.] + }, + :pib => { + :key => :pl_domain_cblocks_linking, + :name => 'All Cblock Linking Domain', + :desc => %Q[The number of unique cblocks with a link to a domain.] + }, + :pjb => { + :key => :pl_domain_juice_cblocks_linking, + :name => 'Followed Cblock Linking Domain', + :desc => %Q[The total number of cblock with followed links to a domain.] + }, + + :upa => { :name => 'Page Authority', :key => :page_authority, @@ -268,8 +360,8 @@ module Constants nil => :source, :lu => :target, } - - + + LinkResponseFields = { :t => { :name => 'Anchor Text', @@ -311,8 +403,8 @@ module Constants LinkResponsePrefixes = { :l => :link, } - - + + AnchorResponseFields = { :t => { :name => 'Anchor Text', @@ -374,10 +466,45 @@ module Constants :atf => :anchor, :atu => :anchor, } - - + + # For values calculated from other values. + # format is [ :unknown_fraction, [ :whole_value, :known_fraction ] ] + # result is data[:unknown_fraction] = data[:whole_value] - data[:known_fraction] + # IFF both data[:whole_value] and data[:known_fraction] are present. + # + # Some calculated values are based on other calculated values, + # so be careful about the ordering of the list. + CalculationKeyMap = [ + [:unfollowed_external_links, [ :all_external_links, :external_links ]], + [:unfollowed_links, [ :links, :juice_links ]], + [:juice_internal_links, [ :juice_links, :external_links ]], + [:internal_links, [ :links, :all_external_links ]], + [:unfollowed_internal_links, [ :internal_links, :juice_internal_links ]], + + [:fq_domain_unfollowed_external_links, [ :fq_domain_all_external_links, :fq_domain_external_links ]], + [:fq_domain_unfollowed_links, [ :fq_domain_links, :fq_domain_juice_links ]], + [:fq_domain_juice_internal_links, [ :fq_domain_juice_links, :fq_domain_external_links ]], + [:fq_domain_internal_links, [ :fq_domain_links, :fq_domain_all_external_links ]], + [:fq_domain_unfollowed_internal_links, [ :fq_domain_internal_links, :fq_domain_juice_internal_links ]], + + [:pl_domain_unfollowed_external_links, [ :pl_domain_all_external_links, :pl_domain_external_links ]], + [:pl_domain_unfollowed_links, [ :pl_domain_links, :pl_domain_juice_links ]], + [:pl_domain_juice_internal_links, [ :pl_domain_juice_links, :pl_domain_external_links ]], + [:pl_domain_internal_links, [ :pl_domain_links, :pl_domain_all_external_links ]], + [:pl_domain_unfollowed_internal_links, [ :pl_domain_internal_links, :pl_domain_juice_internal_links ]], + + [:unfollowed_fq_domains_linking, [ :fq_domains_linking, :juice_fq_domains_linking ]], + [:unfollowed_pl_domains_linking, [ :pl_domains_linking, :juice_pl_domains_linking ]], + [:fq_domain_unfollowed_fq_domains_linking, [ :fq_domain_fq_domains_linking, :fq_domain_juice_fq_domains_linking ]], + [:pl_domain_unfollowed_pl_domains_linking, [ :pl_domain_pl_domains_linking, :pl_domain_juice_pl_domains_linking ]], + [:fq_domain_unfollowed_pl_domains_linking, [ :fq_domain_pl_domains_linking, :fq_domain_juice_pl_domains_linking ]], + [:unfollowed_ips_linking, [ :ips_linking, :juice_ips_linking ]], + [:unfollowed_cblocks_linking, [ :cblocks_linking, :juice_cblocks_linking ]], + [:pl_domain_unfollowed_cblocks_linking, [ :pl_domain_cblocks_linking, :pl_domain_juice_cblocks_linking ]], + [:pl_domain_unfollowed_ips_linking, [ :pl_domain_ips_linking, :pl_domain_juice_ips_linking ]], + ] ResponseFields = {} - + URLResponsePrefixes.each do |prefix, subject| URLResponseFields.each do |k,v| v = v.dup.merge( @@ -408,11 +535,11 @@ module Constants ResponseFields[v[:source]] = v end end - + ResponseFields.keys.each {|k| ResponseFields[ResponseFields[k][:key]] ||= ResponseFields[k] if ResponseFields[k][:key] } - + LongestNameLength = ResponseFields.collect{|k,v|v[:name].length}.max LongestKeyLength = ResponseFields.collect{|k,v|v[:key].to_s.length}.max - + end end diff --git a/lib/linkscape/constants/anchor-metrics.rb b/lib/linkscape/constants/anchor-metrics.rb index ea9a050..e125ce0 100644 --- a/lib/linkscape/constants/anchor-metrics.rb +++ b/lib/linkscape/constants/anchor-metrics.rb @@ -5,65 +5,65 @@ module AnchorMetrics RequestBits = { :text => { :name => 'text', - :flag => 2, + :flag => 2**1, # 2 :desc => %Q[The anchor text term or phrase] }, :record_id => { :name => 'record_id', - :flag => 4, + :flag => 2**2, # 4 :desc => %Q[Internal record ID of the target] }, :internal_pages_linking => { :name => 'Internal Pages Linking', - :flag => 8, + :flag => 2**3, # 8 :desc => %Q[the number of internal pages linking] }, :internal_subdomains_linking => { :name => 'Internal Subdomains Linking', - :flag => 16, + :flag => 2**4, # 16 :desc => %Q[the number of internal subdomains linking] }, :external_pages_linking => { :name => 'External Pages Linking', - :flag => 32, + :flag => 2**5, # 32 :desc => %Q[the number of external pages linking] }, :external_subdomains_linking => { :name => 'External Subdomains Linking', - :flag => 64, + :flag => 2**6, # 64 :desc => %Q[the number of external subdomains linking] }, :external_domains_linking => { :name => 'External Domains Linking', - :flag => 128, + :flag => 2**7, # 128 :desc => %Q[the number of external domains linking] }, :internal_mozrank => { :name => 'Internal mozRank', - :flag => 256, + :flag => 2**8, # 256 :desc => %Q[the sum of mozRank passed along internal links] }, :external_mozrank => { :name => 'External mozRank', - :flag => 512, + :flag => 2**9, # 512 :desc => %Q[the sum of mozRank passed along external links] }, :flags => { :name => 'Flags', - :flag => 1024, + :flag => 2**10, # 1024 :desc => %Q[a flags column] }, } RequestBits[:all] = { - :name => 'All columnts', - :flag => RequestBits.keys.inject(0) {|sum,k| sum + RequestBits[k][:flag]}, + :name => 'All columns', + :flag => RequestBits.keys.inject(0) {|sum,k| sum | RequestBits[k][:flag]}, :desc => %Q[Requests all known columns from the API] } ResponseFlags = { :alt_text => { :name => 'Alt Text', - :flag => 1, + :flag => 2**0, # 1 :desc => %Q[The anchor text is from the alt text of an image] }, } diff --git a/lib/linkscape/constants/link-metrics.rb b/lib/linkscape/constants/link-metrics.rb index 24c830d..1f737a1 100644 --- a/lib/linkscape/constants/link-metrics.rb +++ b/lib/linkscape/constants/link-metrics.rb @@ -5,90 +5,90 @@ module LinkMetrics RequestBits = { :flags => { :name => 'Flags', - :flag => 2, + :flag => 2**1, # 2 :desc => %Q[A bit field indicating a variety of attributes which apply to this link.] }, :text => { :name => 'Anchor Text', - :flag => 4, + :flag => 2**2, # 4 :desc => %Q[The anchor text of the link, including any markup (e.g. image tags with alt text).] }, :mozrank => { :name => 'mozRank Passed', - :flag => 16, + :flag => 2**4, # 16 :desc => %Q[The amount of mozRank passed by the link. Requesting this metric will provide both the pretty 10-point score and the raw score.] } } RequestBits[:all] = { - :name => 'All columnts', - :flag => RequestBits.keys.inject(0) {|sum,k| sum + RequestBits[k][:flag]}, + :name => 'All columns', + :flag => RequestBits.keys.inject(0) {|sum,k| sum | RequestBits[k][:flag]}, :desc => %Q[Requests all known columns from the API] } ResponseFlags = { :no_follow => { :name => 'No Follow', - :flag => 1, + :flag => 2**0, # 1 :desc => %Q[The link in question bore a "rel=nofollow" directive indicating that no juice should flow over the link.] }, :same_subdomain => { :name => 'Same Subdomain', - :flag => 2, + :flag => 2**1, # 2 :desc => %Q[The link is between two pages on the same domain. This is an internal link.] }, :meta_refresh => { :name => 'Meta Refresh', - :flag => 4, + :flag => 2**2, # 4 :desc => %Q[The link is actually a meta refresh from the source page to the target.] }, :same_ip_address => { :name => 'Same IP Address', - :flag => 8, + :flag => 2**3, # 8 :desc => %Q[The link is between two pages hosted on the same IP address, strongly indicating a potential administrative relationship between the two.] }, :same_c_block => { :name => 'Same C-Block', - :flag => 16, + :flag => 2**4, # 16 :desc => %Q[The link is between two pages hosted on the same C Block of IP addresses, indicating a potential administrative relationship between the two.] }, :redirect301 => { :name => '301', - :flag => 64, + :flag => 2**6, # 64 :desc => %Q[The link is a 301 redirect. The source page returned a 301 redirect to our crawler, indicating that the resource was available on the target.] }, :redirect302 => { :name => '302', - :flag => 128, + :flag => 2**7, # 128 :desc => %Q[The link is a 302 redirect. The source page returned a 302 redirect to our crawler, indicating that the resource was temporarily available on the target.] }, :no_script => { :name => 'No Script', - :flag => 256, + :flag => 2**8, # 256 :desc => %Q[The link was located within a noscript html block. This means the link may not have been visible to users using javascript.] }, :off_screen => { :name => 'Off Screen', - :flag => 512, + :flag => 2**9, # 512 :desc => %Q[We determined that the link likely appears offscreen. This means that the link may not have been visible to most users.] }, :meta_no_follow => { :name => 'Meta No Follow', - :flag => 2048, + :flag => 2**11, # 2048 :desc => %Q[The link appeared on a page using a page level (meta) no follow directive. This link passes no juice.] }, :same_root_domain => { :name => 'Same Root Domain', - :flag => 4096, + :flag => 2**12, # 4096 :desc => %Q[The link is between two pages on the same root domain. The link is not internal, but this strongly indicates an administrative relationship between the two pages.] }, :feed_autodiscovery => { :name => 'Feed Autodiscovery', - :flag => 16384, + :flag => 2**14, # 16384 :desc => %Q[The link indicates a syndication feed (e.g. rss or atom) for the source page.] }, :rel_canonical => { :name => 'Rel Canonical', - :flag => 32768, + :flag => 2**15, # 32768 :desc => %Q[The link indicates a canonical form of the page using the rel=canonical directive] } } diff --git a/lib/linkscape/constants/url-metrics.rb b/lib/linkscape/constants/url-metrics.rb index c00b083..f954fb1 100644 --- a/lib/linkscape/constants/url-metrics.rb +++ b/lib/linkscape/constants/url-metrics.rb @@ -5,176 +5,346 @@ module URLMetrics RequestBits = { :title => { :name => 'Title', - :flag => 1, + :flag => 2**0, # 1 :desc => %Q[The title of the page if available. For example: "Request-Response Format"] }, :url => { :name => 'URL', - :flag => 4, + :flag => 2**2, # 4 :desc => %Q[The url of the page. For example: "apiwiki.seomoz.org/Request-Response+Format"] }, :fq_domain => { :name => 'Subdomain', - :flag => 8, + :flag => 2**3, # 8 :desc => %Q[The subdomain of the url. For example: "apiwiki.seomoz.org"] }, :pl_domain => { :name => 'Root Domain', - :flag => 16, + :flag => 2**4, # 16 :desc => %Q[The root domain of the url. For example: "seomoz.org"] }, :external_links => { :name => 'External Links', - :flag => 32, + :flag => 2**5, # 32 :desc => %Q[The number of juice-passing external links to the url.] }, :fq_domain_external_links => { :name => 'Subdomain External Links', - :flag => 64, + :flag => 2**6, # 64 :desc => %Q[The number of juice-passing external links to the subdomain of the url.] }, :pl_domain_external_links => { :name => 'Root Domain External Links', - :flag => 128, + :flag => 2**7, # 128 :desc => %Q[The number of juice-passing external links to the root domain of the url.] }, :juice_links => { :name => 'Juice-Passing Links', - :flag => 256, + :flag => 2**8, # 256 :desc => %Q[The number of juice-passing links (internal or external) to the url.] }, :fq_domains_linking => { :name => 'Subdomains Linking', - :flag => 512, + :flag => 2**9, # 512 :desc => %Q[The number of subdomains with any pages linking to the url.] }, :pl_domains_linking => { :name => 'Root Domains Linking', - :flag => 1024, + :flag => 2**10, # 1024 :desc => %Q[The number of root domains with any pages linking to the url.] }, :links => { :name => 'Links', - :flag => 2048, + :flag => 2**11, # 2048 :desc => %Q[The number of links (juice-passing or not, internal or external) to the url.] }, :fq_domain_links => { :name => 'Subdomain Links', - :flag => 4294967296, + :flag => 2**32, # 4294967296 :desc => %Q[The number of links to any page on the subdomain of the url.] }, :fq_domain_fq_domains_linking => { :name => 'Subdomain Subdomains Linking', - :flag => 4096, + :flag => 2**12, # 4096 :desc => %Q[The number of subdomains with any pages linking to the subdomain of the url.] }, :fq_domain_pl_domains_linking => { :name => 'Subdomain Root Domains Linking', - :flag => 17179869184, + :flag => 2**34, # 17179869184 :desc => %Q[The number of domains with any pages linking to the subdomain of the url.] }, :pl_domain_links => { :name => 'Root Domain Links', - :flag => 8589934592, + :flag => 2**33, # 8589934592 :desc => %Q[The number of links to any page on the root domain of the url.] }, :pl_domain_pl_domains_linking => { :name => 'Root Domain Root Domains Linking', - :flag => 8192, + :flag => 2**13, # 8192 :desc => %Q[The number of root domains with any pages linking to the root domain of the url.] }, :mozrank => { :name => 'mozRank', - :flag => 16384, + :flag => 2**14, # 16384 :desc => %Q[The mozRank of the url. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :fq_domain_mozrank => { :name => 'Subdomain mozRank', - :flag => 32768, + :flag => 2**15, # 32768 :desc => %Q[The mozRank of the subdomain of the url. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :pl_domain_mozrank => { :name => 'Root Domain mozRank', - :flag => 65536, + :flag => 2**16, # 65536 :desc => %Q[The mozRank of the Root Domain of the url. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :moztrust => { :name => 'mozTrust', - :flag => 131072, + :flag => 2**17, # 131072 :desc => %Q[The mozTrust of the url. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :fq_domain_moztrust => { :name => 'Subdomain mozTrust', - :flag => 262144, + :flag => 2**18, # 262144 :desc => %Q[The mozTrust of the subdomain of the url. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :pl_domain_moztrust => { :name => 'Root Domain mozTrust', - :flag => 524288, + :flag => 2**19, # 524288 :desc => %Q[The mozTrust of the root domain of the url. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :external_mozrank => { :name => 'External mozRank', - :flag => 1048576, + :flag => 2**20, # 1048576 :desc => %Q[The portion of the url's mozRank coming from external links. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :fq_domain_external_mozrank_sum_raw => { :name => 'Subdomain External Domain Juice', - :flag => 2097152, + :flag => 2**21, # 2097152 :desc => %Q[The portion of the mozRank of all pages on the subdomain coming from external links. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :pl_domain_external_mozrank_sum_raw => { :name => 'Root Domain External Domain Juice', - :flag => 4194304, + :flag => 2**22, # 4194304 :desc => %Q[The portion of the mozRank of all pages on the root domain coming from external links. Requesting this metric will provide both the pretty 10-point score and the raw score.] - # source.External mozRank sum of all PL Domain Pages (raw) - 9.8334596959365e-11 }, :fq_domain_mozrank_sum_raw => { :name => 'Subdomain Domain Juice', - :flag => 8388608, + :flag => 2**23, # 8388608 :desc => %Q[The mozRank of all pages on the subdomain combined. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :pl_domain_mozrank_sum_raw => { :name => 'Root Domain Domain Juice', - :flag => 16777216, + :flag => 2**24, # 16777216 :desc => %Q[The mozRank of all pages on the root domain combined. Requesting this metric will provide both the pretty 10-point score and the raw score.] }, :canonical_url => { :name => 'Canonical URL', - :flag => 268435456, + :flag => 2**28, # 268435456 :desc => %Q[If the url canaonicalizes to a different form, that canonical form will be available in this field] }, :status => { :name => 'HTTP Status Code', - :flag => 536870912, + :flag => 2**29, # 536870912 :desc => %Q[The HTTP status code recorded by Linkscape for this URL (if available)] }, :page_authority => { :name => 'Page Authority', - :flag => 34359738368, + :flag => 2**35, # 34359738368 :desc => %Q[The page authority of this URL. This will return the pretty 100-point score.] }, :domain_authority => { :name => 'Domain Authority', - :flag => 68719476736, + :flag => 2**36, # 68719476736 :desc => %Q[The page authority of all pages on the root domain. This will return the pretty 100-point score.] }, + + :all_external_links => { + :name => 'All external links page to page', + :flag => 2**39, + :desc => %Q[The number of external links from one page to another (included followed and nofollowed).] + }, + :juice_fq_domains_linking => { + :name => 'Followed Domains Linking Page', + :flag => 2**40, + :desc => %Q[The number of unique domains with followed links to the target url.] + }, + :juice_ips_linking => { + :name => 'Followed IPs Linking', + :flag => 2**41, + :desc => %Q[The number unique IPs with a followable link to a target url.] + }, + :ips_linking => { + :name => 'IPs Linking', + :flag => 2**42, + :desc => %Q[The total number of unique IPs linking to a target url.] + }, + :juice_pl_domains_linking => { + :name => 'Followed Domains to Page', + :flag => 2**43, + :desc => %Q[The number of unique domains with followed links to a given url.] + }, + :cblocks_linking => { + :name => 'All Cblock Linking', + :flag => 2**44, + :desc => %Q[The total number unique cblocks linking to a page.] + }, + :juice_cblocks_linking => { + :name => 'Followed CBLocks Linking', + :flag => 2**45, + :desc => %Q[The total number unique cblocks with followed links to a page.] + }, + :fq_domain_juice_links => { + :name => 'Followed Subdomain Linking Domains', + :flag => 2**46, + :desc => %Q[A count of all unique subdomains with followed links to the target domain.] + }, + :fq_domain_all_external_links => { + :name => 'Subdomain External Links', + :flag => 2**47, + :desc => %Q[The total number (followed and nofollowed) external links to the subdomain of the url.] + }, + :fq_domain_juice_fq_domains_linking => { + :name => 'Followed Subdomain Subdomains Links', + :flag => 2**48, + :desc => %Q[The number of subdomains with followed links to the subdomain of the url.] + }, + :fq_domain_juice_pl_domains_linking => { + :name => 'Followed Domain Subdomains Links', + :flag => 2**49, + :desc => %Q[The number of unique domains with followed links to the subdomain of the url.] + }, + :pl_domain_juice_links => { + :name => 'Followed Root Domain Links', + :flag => 2**50, + :desc => %Q[The total number of followed links (both internal and external) from a page to a domain.] + }, + :pl_domain_all_external_links => { + :name => 'All Root Domain External Links', + :flag => 2**51, + :desc => %Q[The total number of external links (both followed and no-followed) from a page to a domain.] + }, + :pl_domain_juice_pl_domains_linking => { + :name => 'All Followed Root Domains Linking Domain', + :flag => 2**52, + :desc => %Q[The total number of followed root domains linking to the target's domain.] + }, + :pl_domain_ips_linking => { + :name => 'IPs Linking to Domain', + :flag => 2**53, + :desc => %Q[The total number of unique IPs linking to the target's domain.] + }, + :pl_domain_juice_ips_linking => { + :name => 'Followed IPs Linking to Domain', + :flag => 2**54, + :desc => %Q[The total number of unique IPs with followed links to the target's domain.] + }, + :pl_domain_cblocks_linking => { + :name => 'All Cblock Linking Domain', + :flag => 2**55, + :desc => %Q[The number of unique cblocks with a link to a domain.] + }, + :pl_domain_juice_cblocks_linking => { + :name => 'Followed Cblock Linking Domain', + :flag => 2**56, + :desc => %Q[The total number of cblock with followed links to a domain.] + }, + :page_authority_raw => { :name => 'Raw Page Authority', - :flag => 137438953472, + :flag => 2**37, # 137438953472 :desc => %Q[The page authority of this URL. This will return the raw score.] }, :domain_authority_raw => { :name => 'Raw Domain Authority', - :flag => 274877906944, + :flag => 2**38, # 274877906944 :desc => %Q[The page authority of all pages on the root domain. This will return the raw score.] }, + + # The following are calculated values. If you ask for them, + # we'll convert it into a request for the fields they + # depend on. In the Response, we'll then set the key from + # those values. + :unfollowed_external_links => { + :flag => 2**39 | 2**5, + }, + :unfollowed_links => { + :flag => 2**11 | 2**8, + }, + :juice_internal_links => { + :flag => 2**8 | 2**5, + }, + :internal_links => { + :flag => 2**11 | 2**39, + }, + :unfollowed_internal_links => { + :flag => 2**11 | 2**39 | 2**8 | 2**5, + }, + :fq_domain_unfollowed_external_links => { + :flag => 2**47 | 2**6, + }, + :fq_domain_unfollowed_links => { + :flag => 2**32 | 2**46, + }, + :fq_domain_juice_internal_links => { + :flag => 2**46 | 2**6, + }, + :fq_domain_internal_links => { + :flag => 2**32 | 2**47, + }, + :fq_domain_unfollowed_internal_links => { + :flag => 2**32 | 2**47 | 2**46 | 2**6, + }, + :pl_domain_unfollowed_external_links => { + :flag => 2**51 | 2**7, + }, + :pl_domain_unfollowed_links => { + :flag => 2**33 | 2**50, + }, + :pl_domain_juice_internal_links => { + :flag => 2**50 | 2**7, + }, + :pl_domain_internal_links => { + :flag => 2**33 | 2**51, + }, + :pl_domain_unfollowed_internal_links => { + :flag => 2**33 | 2**51 | 2**50 | 2**7, + }, + :unfollowed_fq_domains_linking => { + :flag => 2**9 | 2**40, + }, + :unfollowed_pl_domains_linking => { + :flag => 2**10 | 2**43, + }, + :fq_domain_unfollowed_fq_domains_linking => { + :flag => 2**12 | 2**48, + }, + :pl_domain_unfollowed_pl_domains_linking => { + :flag => 2**13 | 2**52, + }, + :fq_domain_unfollowed_pl_domains_linking => { + :flag => 2**34 | 2**49, + }, + :unfollowed_ips_linking => { + :flag => 2**42 | 2**41, + }, + :unfollowed_cblocks_linking => { + :flag => 2**44 | 2**45, + }, + :pl_domain_unfollowed_cblocks_linking => { + :flag => 2**55 | 2**56, + }, + :pl_domain_unfollowed_ips_linking => { + :flag => 2**53 | 2**54, + }, + + } RequestBits[:all] = { - :name => 'All columnts', - :flag => RequestBits.keys.inject(0) {|sum,k| sum + RequestBits[k][:flag]}, + :name => 'All columns', + :flag => RequestBits.keys.inject(0) {|sum,k| sum | RequestBits[k][:flag]}, :desc => %Q[Requests all known columns from the API] } diff --git a/lib/linkscape/response.rb b/lib/linkscape/response.rb index 3b81d5d..e82e4dd 100644 --- a/lib/linkscape/response.rb +++ b/lib/linkscape/response.rb @@ -32,9 +32,18 @@ def initialize(data, type=nil) elsif Array === @data :array end + @data.symbolize_keys! if Hash === @data @data = @data.collect{|d|ResponseData.new(d)} if Array === @data + if Hash === @data && !type.nil? + Linkscape::Constants::CalculationKeyMap.each do |key, keys| + unless @data[keys[0]].nil? or @data[keys[1]].nil? + @data[key] = @data[keys[0]] - @data[keys[1]] + end + end + end + if @type == :hash subdatas = {} @data.each do |k,v| @@ -75,11 +84,7 @@ def to_s(indent="") printer = Proc.new do |h,prefix| o = "" h.sort{|l,r|l[0].to_s<=>r[0].to_s}.each do |k,v| - field = Linkscape::Constants::ResponseFields[k] v = v.to_s - # v = ((field && field[:bitfield]) ? v.to_a.inspect : v).to_s - #desc = field ? field[:name] : '*'+k.inspect - #o += %Q[%s%-#{Linkscape::Constants::LongestNameLength+15}.#{Linkscape::Constants::LongestNameLength+15}s - %s\n] % [prefix, desc, v] o += %Q[%s%-#{Linkscape::Constants::LongestKeyLength+5}.#{Linkscape::Constants::LongestKeyLength+5}s - %s\n] % [prefix, k, v] end o diff --git a/linkscape.gemspec b/linkscape.gemspec index 18741a1..7d69c11 100644 --- a/linkscape.gemspec +++ b/linkscape.gemspec @@ -1,6 +1,6 @@ # Generated by jeweler # DO NOT EDIT THIS FILE DIRECTLY -# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command +# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec' # -*- encoding: utf-8 -*- Gem::Specification.new do |s| @@ -8,38 +8,36 @@ Gem::Specification.new do |s| s.version = "0.2.7" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= - s.authors = ["Marty Smyth", "Jeff Pollard"] - s.date = %q{2010-08-31} + s.authors = ["Martin Tithonium", "Jeff Pollard", "Bryce Howard"] + s.date = %q{2010-12-09} s.description = %q{Provides an interface to SEOmoz's suite of APIs, including the free and site intelligence APIs.} s.email = %q{api@seomoz.org} s.extra_rdoc_files = [ "LICENSE", - "README.rdoc" + "README.rdoc" ] s.files = [ - ".gitignore", - "LICENSE", - "README.rdoc", - "Rakefile", - "VERSION", - "lib/hash-ext.rb", - "lib/linkscape.rb", - "lib/linkscape/client.rb", - "lib/linkscape/constants.rb", - "lib/linkscape/constants/anchor-metrics.rb", - "lib/linkscape/constants/link-metrics.rb", - "lib/linkscape/constants/url-metrics.rb", - "lib/linkscape/errors.rb", - "lib/linkscape/request.rb", - "lib/linkscape/response.rb", - "lib/linkscape/signer.rb", - "lib/string-ext.rb", - "linkscape.gemspec", - "rails/init.rb", - "test.rb" + "LICENSE", + "README.rdoc", + "Rakefile", + "VERSION", + "lib/hash-ext.rb", + "lib/linkscape.rb", + "lib/linkscape/client.rb", + "lib/linkscape/constants.rb", + "lib/linkscape/constants/anchor-metrics.rb", + "lib/linkscape/constants/link-metrics.rb", + "lib/linkscape/constants/url-metrics.rb", + "lib/linkscape/errors.rb", + "lib/linkscape/request.rb", + "lib/linkscape/response.rb", + "lib/linkscape/signer.rb", + "lib/string-ext.rb", + "linkscape.gemspec", + "rails/init.rb", + "test.rb" ] s.homepage = %q{http://github.com/seomoz/linkscape-gem} - s.rdoc_options = ["--charset=UTF-8"] s.require_paths = ["lib"] s.rubygems_version = %q{1.3.7} s.summary = %q{Provides an interface to the SEOmoz API}