- — @DevDocs + — DevDocs \ `; diff --git a/assets/javascripts/templates/pages/about_tmpl.js b/assets/javascripts/templates/pages/about_tmpl.js index e3142da5fc..600169a5ac 100644 --- a/assets/javascripts/templates/pages/about_tmpl.js +++ b/assets/javascripts/templates/pages/about_tmpl.js @@ -25,14 +25,13 @@ app.templates.aboutPage = function () {
DevDocs is free and open source. It was created by Thibaut Courouble and is operated by freeCodeCamp.
To keep up-to-date with the latest news:
- Copyright 2013–2025 Thibaut Courouble and other contributors
+ Copyright 2013–2026 Thibaut Courouble and other contributors
This software is licensed under the terms of the Mozilla Public License v2.0.
You may obtain a copy of the source code at github.com/freeCodeCamp/devdocs.
For more information, see the COPYRIGHT
diff --git a/assets/javascripts/templates/pages/news_tmpl.js.erb b/assets/javascripts/templates/pages/news_tmpl.js.erb
index 72f4b1dd07..3bcfec32f4 100644
--- a/assets/javascripts/templates/pages/news_tmpl.js.erb
+++ b/assets/javascripts/templates/pages/news_tmpl.js.erb
@@ -2,7 +2,6 @@
app.templates.newsPage = () => `
-For the latest news, follow @DevDocs.
For development updates, follow the project on GitHub.
Happy coding!
diff --git a/assets/javascripts/vendor/prism.js b/assets/javascripts/vendor/prism.js
index 96519c4a8d..d52319c69e 100644
--- a/assets/javascripts/vendor/prism.js
+++ b/assets/javascripts/vendor/prism.js
@@ -1,5 +1,5 @@
/* PrismJS 1.30.0
-https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript+bash+c+cpp+cmake+coffeescript+crystal+d+dart+diff+django+dot+elixir+erlang+go+groovy+java+json+julia+kotlin+latex+lua+markdown+markup-templating+matlab+nginx+nim+nix+ocaml+perl+php+python+qml+r+jsx+ruby+rust+scss+scala+shell-session+sql+tcl+typescript+yaml+zig */
+https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript+bash+c+csharp+cpp+cmake+coffeescript+crystal+d+dart+diff+django+dot+elixir+erlang+gdscript+go+groovy+java+json+julia+kotlin+latex+lua+markdown+markup-templating+matlab+nginx+nim+nix+ocaml+perl+php+python+qml+r+jsx+ruby+rust+scss+scala+shell-session+sql+tcl+typescript+yaml+zig */
/// A functional and reactive JavaScript framework for predictable codeHello World" becomes "Hello")
- node['id'] = node.xpath('text()').to_s.strip
- end
-
- # Fix hashes of index entries so they link to the correct hash on the linked page
- css('table[class^=index-] td[valign=top] > a').each do |node|
- path = node['href'].split('#')[0]
- hash = node.content
-
- # Fix the index entries linking to the Special Parameters page
- # There are multiple index entries that should link to the same paragraph on that page
- # Example: the documentation for "$!" is equal to the documentation for "!"
- if path.downcase.include?('special-parameters')
- if hash.size > 1 && hash[0] == '$'
- hash = hash[1..-1]
- end
+ # The manual marks index targets with empty anchors (), which the
+ # clean_text filter strips. Move their ids onto the enclosing node so the index entries
+ # keep linking to the right paragraph.
+ css('a[id]').each do |node|
+ next unless node.content.strip.empty?
+ parent = node.parent
+ if parent && parent['id'].blank?
+ parent['id'] = node['id']
+ else
+ # Several anchors can share a parent; keep the extra ones alive by giving
+ # them content the clean_text filter doesn't consider empty
+ node.content = ""
end
-
- node['href'] = path + '#' + hash
- end
-
- # Fix index table letter hashes (the "Jump to" hashes)
- css('table[class^=index-] th > a').each do |node|
- node['id'] = node['name']
end
# Remove the rows with a horizontal line in them from the index tables
- css('td[colspan="4"]').remove
+ css('td[colspan="3"]').remove
+
+ # Remove the empty spacer cells of the index tables
+ css('td.printindex-index-entry').each do |node|
+ previous = node.previous_element
+ previous.remove if previous && previous.name == 'td' && previous.content.strip.empty?
+ end
# Remove additional text from menu entry and index entry cells
- css('td[valign=top]').each do |node|
+ css('td.printindex-index-entry, td.printindex-index-section').each do |node|
link = node.at_css('a')
node.children = link unless link.nil?
end
diff --git a/lib/docs/filters/bash/entries.rb b/lib/docs/filters/bash/entries.rb
index c22316a2c2..c146d226f8 100644
--- a/lib/docs/filters/bash/entries.rb
+++ b/lib/docs/filters/bash/entries.rb
@@ -15,13 +15,14 @@ def get_name
name.gsub!(/[[:digit:]]/, '')
end
+ name.remove! '¶'
name.strip
end
def get_type
return 'Manual: Appendices' if name.start_with?('Appendix')
- return 'Manual: Indexes' if at_css('a[rel=up]').content.include?("Index")
+ return 'Manual: Indexes' if at_css('a[rel=up]').try(:content).to_s.include?("Index")
"Manual"
end
@@ -38,7 +39,7 @@ def additional_entries
entries = []
- css('table[class^=index-] td[valign=top] > a').each_slice(2) do |entry_node, section_node|
+ css('td.printindex-index-entry > a').each do |entry_node|
entry_name = entry_node.content
entry_path = entry_node['href']
entries << [entry_name, entry_path, entry_type]
diff --git a/lib/docs/filters/bun/clean_html.rb b/lib/docs/filters/bun/clean_html.rb
index 90dca441d8..43a425c2e3 100644
--- a/lib/docs/filters/bun/clean_html.rb
+++ b/lib/docs/filters/bun/clean_html.rb
@@ -2,7 +2,8 @@ module Docs
class Bun
class CleanHtmlFilter < Filter
def call
- @doc = at_css('section > .w-full')
+ @doc = at_css('#content-area')
+ doc.children = css('#header, #content')
header = at_css('header:has(h1)')
if header
@@ -10,14 +11,15 @@ def call
header.name = 'h1'
end
- css('.CodeBlockTab').remove
- css('.CopyIcon').remove
+ css('*[aria-label="Navigate to header"]', '*[aria-label="Copy the contents from the code block"]').each do |node|
+ node.parent.remove
+ end
+ css('img').remove
css('svg').remove
- css('a:contains("Edit on GitHub")').remove
- css('a:contains("Previous")').remove
- css('a:contains("Next")').remove
-
- css('pre').each do |node|
+
+ css('.code-block *[data-component-part="code-block-header"]').remove
+ css('.code-block', '.code-group').each do |node|
+ node.name = 'pre'
node.content = node.content
node['data-language'] = 'typescript'
node.remove_attribute('style')
@@ -25,6 +27,7 @@ def call
css('.font-mono').each do |node|
node.name = 'code'
+ node.content = node.content
end
css('.font-mono.text-blue-600').each do |node|
diff --git a/lib/docs/filters/bun/entries.rb b/lib/docs/filters/bun/entries.rb
index 7ae0e1b4ae..6703c649dc 100644
--- a/lib/docs/filters/bun/entries.rb
+++ b/lib/docs/filters/bun/entries.rb
@@ -11,12 +11,12 @@ def get_type
end
def additional_entries
- if slug.start_with?('cli')
+ if slug.start_with?('pm/cli')
css('h2[id]').each_with_object [] do |node, entries|
name = get_name + " " + node.content.strip
entries << [name, node['id']]
end
- elsif slug.start_with?('api')
+ elsif slug.start_with?('runtime')
css('h2[id]').each_with_object [] do |node, entries|
name = get_name + ": " + node.content.strip
entries << [name, node['id']]
diff --git a/lib/docs/filters/celery/clean_html.rb b/lib/docs/filters/celery/clean_html.rb
new file mode 100644
index 0000000000..53fc811a4b
--- /dev/null
+++ b/lib/docs/filters/celery/clean_html.rb
@@ -0,0 +1,35 @@
+module Docs
+ class Celery
+ class CleanHtmlFilter < Filter
+ def call
+ # 'This document describes the current stable version of Celery...'
+ css('div.deck').remove
+
+ css('.reference.external.image-reference').remove
+
+ # Table of Contents.
+ css('nav.contents.local', 'table.docutils').remove
+
+ # ¶ anchor links, [source] external links.
+ css('a.headerlink', 'a.toc-backref', '.viewcode-link', 'hr').remove
+
+ css('h1', 'h2', 'h3', 'h4', 'h5', 'h6').each do |node|
+ node.content = node.inner_text
+ end
+
+ css('.highlight-python pre', '.highlight-default pre', '.highlight-pycon pre').each do |node|
+ node.content = node.content
+ node['data-language'] = 'python'
+ node.parent.parent.replace(node)
+ end
+
+ # Lists are wrapped in blockquotes for some reason.
+ css('blockquote ol', 'blockquote ul').each do |node|
+ node.parent.parent.replace(node)
+ end
+
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/celery/entries.rb b/lib/docs/filters/celery/entries.rb
new file mode 100644
index 0000000000..c18a975236
--- /dev/null
+++ b/lib/docs/filters/celery/entries.rb
@@ -0,0 +1,27 @@
+module Docs
+ class Celery
+ class EntriesFilter < Docs::EntriesFilter
+ def get_name
+ at_css('h1').content.gsub(/\P{ASCII}/, '')
+ end
+
+ def get_type
+ if subpath.include?('internals/')
+ 'Internal Module Reference'
+ elsif subpath.include?('reference/')
+ 'API Reference'
+ elsif subpath.include?('userguide/')
+ 'User Guide'
+ else
+ name
+ end
+ end
+
+ def additional_entries
+ css('dl.py dt[id]').each_with_object [] do |node, entries|
+ entries << [node['id'], node['id'], get_type]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/coldfusion/clean_html.rb b/lib/docs/filters/coldfusion/clean_html.rb
new file mode 100644
index 0000000000..99d92a208d
--- /dev/null
+++ b/lib/docs/filters/coldfusion/clean_html.rb
@@ -0,0 +1,91 @@
+module Docs
+ class Coldfusion
+ class CleanHtmlFilter < Filter
+ def call
+ # Listing/category pages (Tags, Functions, a category, or a guide index)
+ # use a different layout; keep their main container as-is after cleanup.
+ @doc = build_root
+
+ # Remove site chrome and interactive widgets.
+ css('nav', 'footer', 'script', 'noscript', '#cfbreak', '.newsletter').remove
+ css('.modal', '.add-example-modal-lg', '.example-modal').remove
+ css('.example-btn', '.copy-btn', '.issuebutton', '.issuecount').remove
+ css('button').remove
+ css('br').remove
+ css('#search2').remove
+
+ # Drop the "Add An Example" / edit / fork affordances.
+ css('a[href*="github.com"]', '#forkme', '#foundeo').remove
+ css('a.label.label-danger').remove # Edit links
+
+ # Clean up the breadcrumb: keep the engine-version labels (they convey
+ # ColdFusion/Lucee/BoxLang availability) but drop navigation links and
+ # the issue tracker widget.
+ if (crumb = at_css('.breadcrumb'))
+ crumb.css('.label-warning').remove
+ crumb.css('.divider').remove
+ crumb.css('a[rel="nofollow"]').remove
+ # Remove navigation breadcrumb items (CFDocs > Functions > cf45 > …)
+ # that are not engine-availability labels.
+ crumb.css('li:not(.pull-right)').each do |li|
+ li.remove unless li.at_css('.label-acf, .label-lucee, .label-boxlang, .label-railo')
+ end
+ end
+
+ # Code blocks: tag them so DevDocs applies CFML syntax highlighting.
+ css('pre.prettyprint', 'pre').each do |node|
+ node.remove_attribute('class')
+ node['data-language'] = 'coldfusion'
+ end
+
+ # Inline code: nothing special needed, but strip prettyprint hints.
+ css('code').each { |node| node.remove_attribute('class') }
+
+ # Unwrap Bootstrap `.container` layout wrappers; DevDocs supplies its own
+ # page width, so these only add centering/padding we don't want.
+ css('.container').each { |node| node.before(node.children).remove }
+
+ # Remove now-empty wrappers left behind by the source template's many
+ # conditional blank lines.
+ css('div', 'p', 'span', 'ul', 'ol').each do |node|
+ node.remove if node.inner_html.strip.empty? && node.element_children.empty?
+ end
+
+ doc
+ end
+
+ # cfdocs splits an entry's content across the `.jumbotron` header (name,
+ # description, syntax), the `.breadcrumb`, and the main `.container`
+ # (arguments, compatibility, links, examples). Merge them into one root.
+ #
+ # NOTE: between filters the document is re-parsed as an HTML *fragment*
+ # (there is no ), so selectors must not depend on `body >`.
+ def build_root
+ # First .jumbotron is the page header; #cfbreak is the trailing
+ # newsletter jumbotron, which we ignore.
+ header = css('.jumbotron').reject { |n| n['id'] == 'cfbreak' }
+ .map { |n| n.at_css('.container') || n }
+ .first
+ breadcrumb = at_css('.breadcrumb')
+
+ # The main content container holds the reference sections. It is a
+ # `.container` that is not the breadcrumb and not inside a jumbotron or
+ # nav. Identify it by the section headings it contains.
+ main = css('.container').find do |node|
+ next false if node.matches?('.breadcrumb')
+ next false if node.ancestors('.jumbotron').any? || node.ancestors('nav').any?
+ node.at_css('h2, .param, .panel') || node.at_css('#examples')
+ end
+
+ root = Nokogiri::HTML.fragment('').at_css('div')
+ root << header.dup if header
+ root << breadcrumb.dup if breadcrumb
+ root << main.dup if main
+
+ # Fall back to the full document/fragment if the expected structure is
+ # missing (e.g. some guide pages).
+ root.element_children.any? ? root : (at_css('body') || doc)
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/coldfusion/entries.rb b/lib/docs/filters/coldfusion/entries.rb
new file mode 100644
index 0000000000..7cdcb4445b
--- /dev/null
+++ b/lib/docs/filters/coldfusion/entries.rb
@@ -0,0 +1,66 @@
+module Docs
+ class Coldfusion
+ class EntriesFilter < Docs::EntriesFilter
+ # Category/listing slugs that aggregate other entries and must not appear
+ # as entries themselves. Categories generally end in "-functions" or
+ # "-tags", but list the fixed index pages explicitly.
+ INDEX_SLUGS = %w(index tags functions all).freeze
+
+ def include_default_entry?
+ entry_page?
+ end
+
+ def get_name
+ if (h1 = at_css('#docname'))
+ h1.content.strip
+ elsif (h1 = at_css('h1'))
+ # Guide pages: use the heading text without anchor noise.
+ h1.content.strip
+ else
+ super
+ end
+ end
+
+ def get_type
+ return 'Guides' if guide_page?
+
+ # Use the second breadcrumb link (Tags / Functions) as the category.
+ crumb = css('.breadcrumb a').map { |a| a.content.strip }
+ if crumb.include?('Tags')
+ 'Tags'
+ elsif crumb.include?('Functions')
+ 'Functions'
+ else
+ 'Guides'
+ end
+ end
+
+ private
+
+ # A real reference entry: a tag or function page. These have a `data-doc`
+ # whose value matches the slug (no spaces) and usually a `#syntax` block.
+ def entry_page?
+ return false if index_slug?
+ return true if guide_page?
+
+ doc_name = at_css('[data-doc]').try(:[], 'data-doc')
+ return false if doc_name.nil?
+ # Category pages have human titles with spaces (e.g. "String Functions").
+ !doc_name.include?(' ')
+ end
+
+ def guide_page?
+ # Guides have no breadcrumb but do have content; they are neither tags
+ # nor functions nor category indexes.
+ return false if index_slug?
+ at_css('.breadcrumb').nil? && at_css('h1')
+ end
+
+ def index_slug?
+ s = slug.to_s.downcase
+ return true if INDEX_SLUGS.include?(s)
+ s.end_with?('-functions', '-tags')
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/core/clean_text.rb b/lib/docs/filters/core/clean_text.rb
index 1deeba01ff..1446868181 100644
--- a/lib/docs/filters/core/clean_text.rb
+++ b/lib/docs/filters/core/clean_text.rb
@@ -6,9 +6,12 @@ class CleanTextFilter < Filter
def call
return html if context[:clean_text] == false
- html.strip!
- while html.gsub!(EMPTY_NODES_RGX, ''); end
- html
+
+ # Clone frozen literal.
+ result = html.dup
+ result.strip!
+ while result.gsub!(EMPTY_NODES_RGX, ''); end
+ result
end
end
end
diff --git a/lib/docs/filters/core/normalize_paths.rb b/lib/docs/filters/core/normalize_paths.rb
index 431c781a06..4daac68bf7 100644
--- a/lib/docs/filters/core/normalize_paths.rb
+++ b/lib/docs/filters/core/normalize_paths.rb
@@ -51,6 +51,8 @@ def normalize_path(path)
"#{path}index"
elsif path.end_with? '.html'
path[0..-6]
+ elsif path.end_with? '.md'
+ path[0..-4]
else
path
end
diff --git a/lib/docs/filters/couchdb/clean_html.rb b/lib/docs/filters/couchdb/clean_html.rb
new file mode 100644
index 0000000000..7c30aea4fa
--- /dev/null
+++ b/lib/docs/filters/couchdb/clean_html.rb
@@ -0,0 +1,29 @@
+module Docs
+ class Couchdb
+ class CleanHtmlFilter < Filter
+ def call
+ css('.section-number').remove
+ css('.headerlink').remove
+
+ css('.sig-name').each do |node|
+ node.name = 'code'
+ end
+
+ css('pre').each do |node|
+ node.content = node.content.strip
+
+ classes = node.parent.parent.classes
+ if classes.include? 'highlight-bash'
+ node['data-language'] = 'bash'
+ else
+ node['data-language'] = 'javascript'
+ end
+
+ node.parent.parent.replace(node)
+ end
+
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/couchdb/entries.rb b/lib/docs/filters/couchdb/entries.rb
new file mode 100644
index 0000000000..3d03a9616e
--- /dev/null
+++ b/lib/docs/filters/couchdb/entries.rb
@@ -0,0 +1,53 @@
+module Docs
+ class Couchdb
+ class EntriesFilter < Docs::EntriesFilter
+ SLUG_MAP = {
+ 'api' => 'API',
+ 'json' => 'JSON Structures',
+ 'cluster' => 'Cluster Management',
+ 'replication' => 'Replication',
+ 'maintenance' => 'Maintenance',
+ 'partitioned' => 'Partitioned Databases'
+ }
+
+ def get_name
+ at_css('h1').content.gsub(/\P{ASCII}/, '').split('.').last
+ end
+
+ def get_type
+ if slug.start_with?('ddocs/views')
+ 'Views'
+ elsif slug.start_with?('ddocs')
+ 'Design Documents'
+ else
+ SLUG_MAP[slug[/^(.+?)[-\/]/, 1]] || name
+ end
+ end
+
+ def additional_entries
+ needs_breakup = [
+ 'JSON Structure Reference',
+ 'Design Documents',
+ 'Partitioned Databases'
+ ]
+
+ if needs_breakup.include?(name)
+ entries = []
+
+ css('section > section').each do |node|
+ h2 = node.at_css('h2')
+
+ if h2.present?
+ name = node.at_css('h2').content.split('.').last
+ entries << [name, node['id']]
+ end
+ end
+
+ entries
+ else
+ []
+ end
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/cyclejs/clean_html.rb b/lib/docs/filters/cyclejs/clean_html.rb
new file mode 100644
index 0000000000..cfafee84b9
--- /dev/null
+++ b/lib/docs/filters/cyclejs/clean_html.rb
@@ -0,0 +1,23 @@
+module Docs
+ class Cyclejs
+ class CleanHtmlFilter < Filter
+ def call
+ return "Cycle.js
htmx
") if root_page?
css('div:contains("NEWS:")').remove
diff --git a/lib/docs/filters/http/clean_html.rb b/lib/docs/filters/http/clean_html.rb
index 9c5abe3e35..c63e7743c5 100644
--- a/lib/docs/filters/http/clean_html.rb
+++ b/lib/docs/filters/http/clean_html.rb
@@ -25,8 +25,18 @@ def mdn
end
def ietf
- raise "#{slug} is obsolete!" if at_css('.meta-info *:contains("Obsoleted by")')
- @doc = at_css('.draftcontent')
+ # datatracker serves two layouts: a legacy plain-text rendering wrapped
+ # in .rfcmarkup (older RFCs) and a modern structured HTML rendering
+ # wrapped in .rfchtml (RFCs with xml2rfc v3 sources, e.g. RFC 9110+).
+ doc['class'].to_s.include?('rfchtml') ? ietf_html : ietf_text
+ end
+
+ def ietf_html
+ css('.pilcrow', '.noprint').remove
+ end
+
+ def ietf_text
+ @doc = at_css('.rfcmarkup')
doc.child.remove while doc.child.name != 'pre'
css('span.grey', '.invisible', '.noprint', 'a[href^="#page-"]').remove
diff --git a/lib/docs/filters/http/entries.rb b/lib/docs/filters/http/entries.rb
index 0bc64538c4..0686c17276 100644
--- a/lib/docs/filters/http/entries.rb
+++ b/lib/docs/filters/http/entries.rb
@@ -3,7 +3,9 @@ class Http
class EntriesFilter < Docs::EntriesFilter
def get_name
if current_url.host == 'datatracker.ietf.org'
- name = at_css('h1').content
+ # The modern (.rfchtml) layout has two h1s: the RFC number and the
+ # title. The legacy (.rfcmarkup) layout has a single title h1.
+ name = (at_css('h1#title') || at_css('h1')).content
name.remove! %r{\A.+\:}
name.remove! %r{\A.+\-\-}
name = 'WebDAV' if name.include?('WebDAV')
@@ -24,13 +26,13 @@ def get_name
def get_type
return name if current_url.host == 'datatracker.ietf.org'
- if slug.start_with?('Headers/Content-Security-Policy')
+ if slug.start_with?('Reference/Headers/Content-Security-Policy')
'CSP'
- elsif slug.start_with?('Headers')
+ elsif slug.start_with?('Reference/Headers')
'Headers'
- elsif slug.start_with?('Methods')
+ elsif slug.start_with?('Reference/Methods')
'Methods'
- elsif slug.start_with?('Status')
+ elsif slug.start_with?('Reference/Status')
'Status'
elsif slug.start_with?('Basics_of_HTTP')
'Guides: Basics'
@@ -86,15 +88,31 @@ def rfc
LEVEL_3 = /\A(\d+)\.\d+\.\d+\z/
LEVEL_4 = /\A(\d+)\.\d+\.\d+\.\d+\z/
+ # Returns ordered [section-id, heading-text] pairs for each numbered
+ # section, handling both datatracker layouts: the legacy text rendering
+ # puts the section id on the heading element itself, while the modern HTML
+ # rendering wraps each section in
+ # with per-line anchors. Flatten each block to its plain text and tag
+ # the language so DevDocs can re-highlight it.
+ css('div.highlight').each do |node|
+ language = node['class'][/language-(\w+)/, 1]
+ pre = node.at_css('pre')
+ next unless pre
+
+ pre['data-language'] = language if language
+ pre.content = pre.at_css('code').content
+ node.replace(pre)
+ end
+
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/maplibre_gl/entries.rb b/lib/docs/filters/maplibre_gl/entries.rb
new file mode 100644
index 0000000000..61b36da220
--- /dev/null
+++ b/lib/docs/filters/maplibre_gl/entries.rb
@@ -0,0 +1,52 @@
+module Docs
+ class MaplibreGl
+ class EntriesFilter < Docs::EntriesFilter
+ # Sections of a TypeDoc class page whose members are worth indexing
+ # individually.
+ MEMBER_SECTIONS = %w(Constructors Properties Methods Events Accessors).freeze
+
+ def get_name
+ at_css('h1').content.strip
+ end
+
+ def get_type
+ # DevDocs lower-cases paths, so match against the lower-cased segments.
+ case path
+ when %r{api/classes/} then 'Classes'
+ when %r{api/functions/} then 'Functions'
+ when %r{api/type-aliases/} then 'Type Aliases'
+ when %r{api/interfaces/} then 'Interfaces'
+ when %r{api/enumerations/} then 'Enumerations'
+ when %r{api/variables/} then 'Variables'
+ when %r{\Aguides/} then 'Guides'
+ else 'Miscellaneous'
+ end
+ end
+
+ def additional_entries
+ return [] unless class_page?
+
+ entries = []
+ section = nil
+
+ css('h2, h3').each do |node|
+ if node.name == 'h2'
+ section = node.content.strip
+ elsif MEMBER_SECTIONS.include?(section)
+ member = node.content.strip.sub(/\(\)\z/, '').sub(/\?\z/, '')
+ next if member.empty?
+ entries << ["#{get_name}.#{member}", "#{path}##{node['id']}"]
+ end
+ end
+
+ entries
+ end
+
+ private
+
+ def class_page?
+ path.include?('api/classes/')
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/mdn/compat_tables.rb b/lib/docs/filters/mdn/compat_tables.rb
index 162fd3b76c..5b1db94422 100644
--- a/lib/docs/filters/mdn/compat_tables.rb
+++ b/lib/docs/filters/mdn/compat_tables.rb
@@ -138,77 +138,52 @@ def add_data_to_entry(json, entry)
version_removed = []
notes = []
- if json
- if json.is_a?(Array)
- json.each do |element|
-
- if element['version_added']
- version_added.push(element['version_added'])
- else
- version_added.push(false)
- end
-
- if element['version_removed']
- version_removed.push(element['version_removed'])
- else
- version_removed.push(false)
- end
-
- if element['notes']
- notes.push(element['notes'])
- else
- notes.push(false)
- end
-
- end
- else
- version_added.push(json['version_added'])
- version_removed.push(json['version_removed'])
- notes.push(json['notes'])
- end
+ if !json
+ format_string = "…lots of spans…"
+ (json.is_a?(Array) ? json : [json]).each do |element|
+ version = element['version_added']
+ if version.is_a?(String) and element['release_date']
+ format_string = " "
+ format_string = " " if element['release_date'] > Time.now.iso8601
+ version_added.push("#{version}")
+ elsif version == 'preview'
+ format_string = " "
+ version_added.push(version)
+ elsif version.is_a?(String)
+ format_string = " "
+ version_added.push(version)
+ elsif version == true
+ format_string = " "
+ version_added.push('Yes')
+ else
+ format_string = " "
+ version_added.push('No')
end
+ version_removed.push(element['version_removed'] || false)
+ notes.push(element['notes'] || false)
+ end
- if version_removed[0]
- format_string = " "
- elsif version_added[0] == 'No'
- format_string = " "
- elsif version_added[0] == '?'
- format_string = " "
+ for value in (0..version_added.length-1) do
+ if version_removed[value]
+ version_string = "#{version_added[value]}–#{version_removed[value]}"
else
- format_string = " "
+ version_string = version_added[value]
end
- for value in (0..version_added.length-1) do
- if version_removed[value]
- version_string = "#{version_added[value]}–#{version_removed[value]}"
- else
- version_string = version_added[value]
- end
-
- if notes[value]
- format_string += " "
-
- else
- format_string = ""
- else
- format_string += "#{version_string}
#{notes[value]}#{version_string}"
- end
+ if notes[value]
+ format_string += ""
+ else
+ format_string += "#{version_string}
#{notes[value]}#{version_string}"
end
-
- format_string += "
+ Celery is licensed under The BSD License (3 Clause, also known as the new BSD license). The license is an OSI approved Open Source license and is GPL-compatible.
+ HTML
+
+ def get_latest_version(opts)
+ tags = get_github_tags('celery', 'celery', opts)
+ tags[0]['name'][1..-1]
+ end
+ end
+end
diff --git a/lib/docs/scrapers/coldfusion.rb b/lib/docs/scrapers/coldfusion.rb
new file mode 100644
index 0000000000..ce827d9eda
--- /dev/null
+++ b/lib/docs/scrapers/coldfusion.rb
@@ -0,0 +1,58 @@
+module Docs
+ class Coldfusion < UrlScraper
+ self.name = 'ColdFusion'
+ self.slug = 'coldfusion'
+ self.type = 'simple'
+ self.base_url = 'https://cfdocs.org/'
+ self.root_path = 'index.cfm'
+ self.links = {
+ home: 'https://cfdocs.org/',
+ code: 'https://github.com/foundeo/cfdocs'
+ }
+ self.release = '2026-04-30'
+
+ html_filters.push 'coldfusion/entries', 'coldfusion/clean_html'
+
+ options[:root_title] = 'ColdFusion'
+
+ # cfdocs links categories with an encoded dash (e.g. /array%2Dfunctions);
+ # decode and clean those so entry paths look like /array-functions.
+ options[:decode_and_clean_paths] = true
+
+ # cfdocs.org renders a page for every tag/function/guide at the site root,
+ # e.g. /hash or /cfhtmltopdf. Category "listing" pages (such as /tags,
+ # /functions and /array-functions) are crawled to discover entries, but the
+ # Entries filter excludes them from the index.
+ #
+ # Skip site chrome, utilities, reports and other non-reference pages.
+ options[:skip] = %w(
+ 404.cfm contributors.cfm trycf.cfm ucase.cfm llms.cfm
+ how-to-contribute opensearch.xml robots.txt)
+
+ options[:skip_patterns] = [
+ /\Aassets\b/,
+ /\Areports\b/,
+ /\Autilities\b/,
+ /\Aslack\b/,
+ /openimage/,
+ /\.json\z/,
+ /\.png\z/,
+ /\.ico\z/,
+ /\.xml\z/,
+ /\.css\z/,
+ /\.js\z/
+ ]
+
+ options[:attribution] = <<-HTML
+ © 2012–present Foundeo, Inc. and the CFDocs contributors.
+ Licensed under the MIT License.
+ ColdFusion is a trademark of Adobe Systems Incorporated.
+ HTML
+
+ def get_latest_version(opts)
+ # CFDocs is continuously updated and has no formal version number; use the
+ # date of the latest commit as a proxy version.
+ get_latest_github_commit_date('foundeo', 'cfdocs', opts)
+ end
+ end
+end
diff --git a/lib/docs/scrapers/couchdb.rb b/lib/docs/scrapers/couchdb.rb
new file mode 100644
index 0000000000..a077e195a1
--- /dev/null
+++ b/lib/docs/scrapers/couchdb.rb
@@ -0,0 +1,39 @@
+module Docs
+ class Couchdb < UrlScraper
+ self.name = 'CouchDB'
+ self.type = 'couchdb'
+ self.root_path = 'index.html'
+
+ self.links = {
+ home: 'https://couchdb.apache.org/',
+ code: 'https://github.com/apache/couchdb'
+ }
+
+ html_filters.push 'couchdb/clean_html', 'couchdb/entries'
+
+ options[:container] = 'div[itemprop=articleBody]'
+ options[:only_patterns] = [
+ /api\//,
+ /cluster\//,
+ /ddocs\//,
+ /replication\//,
+ /maintenance\//,
+ /partitioned-dbs\//,
+ /json\-structure*/
+ ]
+ options[:rate_limit] = 50 # Docs are subject to Cloudflare limiters.
+ options[:attribution] = <<-HTML
+ Copyright © 2025 The Apache Software Foundation — Licensed under the Apache License 2.0
+ HTML
+
+ version '3.5' do
+ self.release = '3.5.1'
+ self.base_url = "https://docs.couchdb.org/en/#{self.release}"
+ end
+
+ def get_latest_version(opts)
+ doc = fetch_doc('https://couchdb.apache.org/', opts)
+ doc.at_css('.download-pane > h2').content.split(' ').last
+ end
+ end
+end
diff --git a/lib/docs/scrapers/crystal.rb b/lib/docs/scrapers/crystal.rb
index 09e62a4558..0534c8525d 100644
--- a/lib/docs/scrapers/crystal.rb
+++ b/lib/docs/scrapers/crystal.rb
@@ -2,7 +2,7 @@ module Docs
class Crystal < UrlScraper
include MultipleBaseUrls
self.type = 'crystal'
- self.release = '1.18.0'
+ self.release = '1.20.0'
self.base_urls = [
"https://crystal-lang.org/api/#{release}/",
"https://crystal-lang.org/reference/#{release[0..2]}/",
@@ -37,7 +37,7 @@ def initial_urls
HTML
else
<<-HTML
- © 2012–2025 Manas Technology Solutions.
+ © 2012–2026 Manas Technology Solutions.
Licensed under the Apache License, Version 2.0.
HTML
end
diff --git a/lib/docs/scrapers/cyclejs.rb b/lib/docs/scrapers/cyclejs.rb
new file mode 100644
index 0000000000..abaa3c9a7d
--- /dev/null
+++ b/lib/docs/scrapers/cyclejs.rb
@@ -0,0 +1,84 @@
+module Docs
+ class Cyclejs < UrlScraper
+ self.name = 'Cycle.js'
+ self.slug = 'cyclejs'
+ self.type = 'cyclejs'
+ self.release = '23.1.0'
+ self.base_url = 'https://cycle.js.org/'
+ self.root_path = 'index.html'
+ self.initial_paths = %w(
+ getting-started.html
+ model-view-intent.html
+ streams.html
+ drivers.html
+ components.html
+ basic-examples.html
+ dialogue.html
+ releases.html
+ api/index.html
+ api/run.html
+ api/rxjs-run.html
+ api/most-run.html
+ api/dom.html
+ api/html.html
+ api/http.html
+ api/history.html
+ api/isolate.html
+ api/state.html
+ )
+
+ self.links = {
+ home: 'https://cycle.js.org/',
+ code: 'https://github.com/cyclejs/cyclejs'
+ }
+
+ html_filters.push 'cyclejs/clean_html', 'cyclejs/entries'
+
+ options[:only_patterns] = [
+ /\Aindex\.html\z/,
+ /\Agetting-started\.html\z/,
+ /\Amodel-view-intent\.html\z/,
+ /\Astreams\.html\z/,
+ /\Adrivers\.html\z/,
+ /\Acomponents\.html\z/,
+ /\Abasic-examples\.html\z/,
+ /\Adialogue\.html\z/,
+ /\Areleases\.html\z/,
+ /\Aapi\//
+ ]
+
+ options[:download_images] = false
+ options[:attribution] = <<-HTML
+ © 2014–present Cycle.js contributors.
+ Licensed under the MIT License.
+ HTML
+
+ def get_latest_version(opts)
+ get_npm_version('@cycle/dom', opts)
+ end
+
+ private
+
+ def parse(response)
+ document = Parser.new(response.body).html
+ markdown = document.at_css('script#markdown')
+
+ return super unless markdown
+
+ html = markdown_renderer.render(markdown.content.strip)
+ title = document.at_css('title').try(:content).try(:strip)
+ [Parser.new("
+ © 2012-2026 The Elixir Team
Licensed under the Apache License, Version 2.0.
HTML
def initial_urls
- [ "https://hexdocs.pm/elixir/#{self.class.release}/introduction.html",
- "https://hexdocs.pm/eex/#{self.class.release}/EEx.html",
- "https://hexdocs.pm/ex_unit/#{self.class.release}/ExUnit.html",
- "https://hexdocs.pm/iex/#{self.class.release}/IEx.html",
- "https://hexdocs.pm/logger/#{self.class.release}/Logger.html",
- "https://hexdocs.pm/mix/#{self.class.release}/Mix.html" ]
+ [ "https://elixir.hexdocs.pm/#{self.class.release}/introduction.html",
+ "https://eex.hexdocs.pm/#{self.class.release}/EEx.html",
+ "https://ex-unit.hexdocs.pm/#{self.class.release}/ExUnit.html",
+ "https://iex.hexdocs.pm/#{self.class.release}/IEx.html",
+ "https://logger.hexdocs.pm/#{self.class.release}/Logger.html",
+ "https://mix.hexdocs.pm/#{self.class.release}/Mix.html" ]
+ end
+
+ version '1.20' do
+ self.release = '1.20.1'
+ self.base_urls = [
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/"
+ ]
end
version '1.18' do
self.release = '1.18.1'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/"
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/"
]
end
version '1.17' do
self.release = '1.17.2'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/"
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/"
]
end
version '1.16' do
self.release = '1.16.3'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/"
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/"
]
end
@@ -71,12 +83,12 @@ def initial_urls
version '1.15' do
self.release = '1.15.4'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -84,12 +96,12 @@ def initial_urls
version '1.14' do
self.release = '1.14.1'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -97,12 +109,12 @@ def initial_urls
version '1.13' do
self.release = '1.13.4'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -110,12 +122,12 @@ def initial_urls
version '1.12' do
self.release = '1.12.0'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -123,12 +135,12 @@ def initial_urls
version '1.11' do
self.release = '1.11.2'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -136,12 +148,12 @@ def initial_urls
version '1.10' do
self.release = '1.10.4'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -149,12 +161,12 @@ def initial_urls
version '1.9' do
self.release = '1.9.4'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -162,12 +174,12 @@ def initial_urls
version '1.8' do
self.release = '1.8.2'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -175,12 +187,12 @@ def initial_urls
version '1.7' do
self.release = '1.7.4'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -188,12 +200,12 @@ def initial_urls
version '1.6' do
self.release = '1.6.6'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -201,12 +213,12 @@ def initial_urls
version '1.5' do
self.release = '1.5.3'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -214,12 +226,12 @@ def initial_urls
version '1.4' do
self.release = '1.4.5'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
@@ -227,18 +239,18 @@ def initial_urls
version '1.3' do
self.release = '1.3.4'
self.base_urls = [
- "https://hexdocs.pm/elixir/#{release}/",
- "https://hexdocs.pm/eex/#{release}/",
- "https://hexdocs.pm/ex_unit/#{release}/",
- "https://hexdocs.pm/iex/#{release}/",
- "https://hexdocs.pm/logger/#{release}/",
- "https://hexdocs.pm/mix/#{release}/",
+ "https://elixir.hexdocs.pm/#{release}/",
+ "https://eex.hexdocs.pm/#{release}/",
+ "https://ex-unit.hexdocs.pm/#{release}/",
+ "https://iex.hexdocs.pm/#{release}/",
+ "https://logger.hexdocs.pm/#{release}/",
+ "https://mix.hexdocs.pm/#{release}/",
'https://elixir-lang.org/getting-started/'
]
end
def get_latest_version(opts)
- doc = fetch_doc('https://hexdocs.pm/elixir/api-reference.html', opts)
+ doc = fetch_doc('https://elixir.hexdocs.pm/api-reference.html', opts)
doc.at_css('.sidebar-projectVersion').content.strip[1..-1]
end
end
diff --git a/lib/docs/scrapers/es_toolkit.rb b/lib/docs/scrapers/es_toolkit.rb
index 85079723a0..0b5f26bba2 100644
--- a/lib/docs/scrapers/es_toolkit.rb
+++ b/lib/docs/scrapers/es_toolkit.rb
@@ -7,14 +7,15 @@ class EsToolkit < FileScraper
code: "https://github.com/toss/es-toolkit",
home: "https://es-toolkit.slash.page",
}
+ self.release = '1.45.1'
options[:attribution] = <<-HTML
- © 2024-2025, Viva Republica
+ © 2024-2026, Viva Republica
Licensed under the MIT License.
HTML
def get_latest_version(opts)
- get_github_tags("toss", "es-toolkit", opts).first["name"]
+ get_github_tags("toss", "es-toolkit", opts).first["name"][1..]
end
def build_pages(&block)
diff --git a/lib/docs/scrapers/eslint.rb b/lib/docs/scrapers/eslint.rb
index af71cf4a8d..6cd6c90196 100644
--- a/lib/docs/scrapers/eslint.rb
+++ b/lib/docs/scrapers/eslint.rb
@@ -2,7 +2,7 @@ module Docs
class Eslint < UrlScraper
self.name = 'ESLint'
self.type = 'simple'
- self.release = '9.25.0'
+ self.release = '10.4.0'
self.base_url = 'https://eslint.org/docs/latest/'
self.root_path = '/'
self.links = {
diff --git a/lib/docs/scrapers/express.rb b/lib/docs/scrapers/express.rb
index ff6514b5fb..887787e8f1 100644
--- a/lib/docs/scrapers/express.rb
+++ b/lib/docs/scrapers/express.rb
@@ -28,7 +28,7 @@ class Express < UrlScraper
HTML
version do
- self.release = '5.1.0'
+ self.release = '5.2.1'
self.root_path = '5x/api.html'
end
diff --git a/lib/docs/scrapers/fastapi.rb b/lib/docs/scrapers/fastapi.rb
index d5b3f4e5d5..7fd8a787f8 100644
--- a/lib/docs/scrapers/fastapi.rb
+++ b/lib/docs/scrapers/fastapi.rb
@@ -2,7 +2,7 @@ module Docs
class Fastapi < UrlScraper
self.name = 'FastAPI'
self.type = 'fastapi'
- self.release = '0.115.6'
+ self.release = '0.136.3'
self.base_url = 'https://fastapi.tiangolo.com/'
self.root_path = '/'
self.links = {
diff --git a/lib/docs/scrapers/fish.rb b/lib/docs/scrapers/fish.rb
index d74ed975ab..fb3aa669d3 100644
--- a/lib/docs/scrapers/fish.rb
+++ b/lib/docs/scrapers/fish.rb
@@ -12,12 +12,60 @@ class Fish < UrlScraper
# https://fishshell.com/docs/current/license.html
options[:attribution] = <<-HTML
- © 2005-2009 Axel Liljencrantz, 2009-2025 fish-shell contributors
+ © 2005-2009 Axel Liljencrantz, 2009-2026 fish-shell contributors
Licensed under the GNU General Public License, version 2.
HTML
+ version '4.7' do
+ self.release = '4.7.1'
+ self.base_url = "https://fishshell.com/docs/#{version}/"
+
+ options[:skip].concat %w(genindex.html relnotes.html)
+ html_filters.push 'sphinx/clean_html', 'fish/clean_html_sphinx', 'fish/entries_sphinx'
+ end
+
+ version '4.6' do
+ self.release = '4.6.0'
+ self.base_url = "https://fishshell.com/docs/#{version}/"
+
+ options[:skip].concat %w(genindex.html relnotes.html)
+ html_filters.push 'sphinx/clean_html', 'fish/clean_html_sphinx', 'fish/entries_sphinx'
+ end
+
+ version '4.5' do
+ self.release = '4.5.0'
+ self.base_url = "https://fishshell.com/docs/#{version}/"
+
+ options[:skip].concat %w(genindex.html relnotes.html)
+ html_filters.push 'sphinx/clean_html', 'fish/clean_html_sphinx', 'fish/entries_sphinx'
+ end
+
+ version '4.4' do
+ self.release = '4.4.0'
+ self.base_url = "https://fishshell.com/docs/#{version}/"
+
+ options[:skip].concat %w(genindex.html relnotes.html)
+ html_filters.push 'sphinx/clean_html', 'fish/clean_html_sphinx', 'fish/entries_sphinx'
+ end
+
+ version '4.3' do
+ self.release = '4.3.3'
+ self.base_url = "https://fishshell.com/docs/#{version}/"
+
+ options[:skip].concat %w(genindex.html relnotes.html)
+ html_filters.push 'sphinx/clean_html', 'fish/clean_html_sphinx', 'fish/entries_sphinx'
+ end
+
+ version '4.2' do
+ self.release = '4.2.1'
+ self.base_url = "https://fishshell.com/docs/#{version}/"
+
+ options[:skip].concat %w(genindex.html relnotes.html)
+ html_filters.push 'sphinx/clean_html', 'fish/clean_html_sphinx', 'fish/entries_sphinx'
+ end
+
version '4.1' do
- self.release = '4.1.0'
+ self.release = '4.1.2'
self.base_url = "https://fishshell.com/docs/#{version}/"
options[:skip].concat %w(genindex.html relnotes.html)
diff --git a/lib/docs/scrapers/git.rb b/lib/docs/scrapers/git.rb
index 2bda8d96dd..fcbcf4ffd4 100644
--- a/lib/docs/scrapers/git.rb
+++ b/lib/docs/scrapers/git.rb
@@ -1,7 +1,7 @@
module Docs
class Git < UrlScraper
self.type = 'git'
- self.release = '2.51.0'
+ self.release = '2.54.0'
self.base_url = 'https://git-scm.com/docs'
self.initial_paths = %w(
/git.html
@@ -41,12 +41,12 @@ class Git < UrlScraper
options[:container] = '#content'
options[:only_patterns] = [/\A\/[^\/]+\z/]
- options[:skip] = %w(/howto-index.html)
+ options[:skip] = %w(/api-index /howto-index)
# https://github.com/git/git?tab=License-1-ov-file#readme
# NOT https://github.com/git/git-scm.com/blob/gh-pages/MIT-LICENSE.txt
options[:attribution] = <<-HTML
- © 2005–2025 Linus Torvalds and others
+ © 2005–2026 Linus Torvalds and others
Licensed under the GNU General Public License version 2.
HTML
diff --git a/lib/docs/scrapers/github.rb b/lib/docs/scrapers/github.rb
index dcadcec865..144dbd7802 100644
--- a/lib/docs/scrapers/github.rb
+++ b/lib/docs/scrapers/github.rb
@@ -23,7 +23,7 @@ def parse(response)
&.first
parsed = JSON.parse(embedded_json)
- [parsed['payload']['blob']['richText'], parsed['title']]
+ [parsed['payload']['codeViewBlobRoute']['richText'], parsed['title']]
end
end
end
diff --git a/lib/docs/scrapers/go.rb b/lib/docs/scrapers/go.rb
index 4f12e3cc98..717aff483f 100644
--- a/lib/docs/scrapers/go.rb
+++ b/lib/docs/scrapers/go.rb
@@ -1,7 +1,7 @@
module Docs
class Go < UrlScraper
self.type = 'go'
- self.release = '1.25.1'
+ self.release = '1.26.0'
self.base_url = 'https://golang.org/pkg/'
self.links = {
home: 'https://golang.org/',
@@ -10,7 +10,7 @@ class Go < UrlScraper
# Run godoc locally, since https://golang.org/pkg/ redirects to https://pkg.go.dev/std with rate limiting / scraping protection.
- # podman run --net host --rm -it docker.io/golang:1.25.1
+ # podman run --net host --rm -it docker.io/golang:1.26.0
#podman# go install golang.org/x/tools/cmd/godoc@latest
#podman# rm -r /usr/local/go/test/
#podman# godoc -http 0.0.0.0:6060 -v
@@ -18,8 +18,8 @@ class Go < UrlScraper
# or using alpine
# podman run --net host --rm -it alpine:latest
#podman# apk add curl
- #podman# curl -LO https://go.dev/dl/go1.25.1.linux-amd64.tar.gz
- #podman# tar xf go1.25.1.linux-amd64.tar.gz
+ #podman# curl -LO https://go.dev/dl/go1.26.0.linux-amd64.tar.gz
+ #podman# tar xf go1.26.0.linux-amd64.tar.gz
#podman# go/bin/go install golang.org/x/tools/cmd/godoc@latest
#podman# /root/go/bin/godoc -http 0.0.0.0:6060 -v
diff --git a/lib/docs/scrapers/godot.rb b/lib/docs/scrapers/godot.rb
index 3c7995479a..970d53f8a5 100644
--- a/lib/docs/scrapers/godot.rb
+++ b/lib/docs/scrapers/godot.rb
@@ -21,6 +21,12 @@ class Godot < UrlScraper
Licensed under the Creative Commons Attribution Unported License v3.0.
HTML
+ version '4.7' do
+ self.release = '4.7.0'
+ self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
+ html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'
+ end
+
version '4.2' do
self.release = '4.2.2'
self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
diff --git a/lib/docs/scrapers/haproxy.rb b/lib/docs/scrapers/haproxy.rb
index 5f87168732..6e48add6b0 100644
--- a/lib/docs/scrapers/haproxy.rb
+++ b/lib/docs/scrapers/haproxy.rb
@@ -16,12 +16,27 @@ class Haproxy < UrlScraper
options[:follow_links] = false
options[:attribution] = <<-HTML
- © 2024 Willy Tarreau, HAProxy contributors
+ © 2025 Willy Tarreau, HAProxy contributors
Licensed under the GNU General Public License version 2.
HTML
+ version '3.3' do
+ self.release = '3.3.0'
+ self.base_url = "https://docs.haproxy.org/#{self.version}/"
+ end
+
+ version '3.2' do
+ self.release = '3.2.9'
+ self.base_url = "https://docs.haproxy.org/#{self.version}/"
+ end
+
+ version '3.1' do
+ self.release = '3.1.10'
+ self.base_url = "https://docs.haproxy.org/#{self.version}/"
+ end
+
version '3.0' do
- self.release = '3.0.0'
+ self.release = '3.0.12'
self.base_url = "https://docs.haproxy.org/#{self.version}/"
end
diff --git a/lib/docs/scrapers/haskell.rb b/lib/docs/scrapers/haskell.rb
index 8d822ece0f..d0aeda1456 100644
--- a/lib/docs/scrapers/haskell.rb
+++ b/lib/docs/scrapers/haskell.rb
@@ -78,10 +78,9 @@ class Haskell < UrlScraper
end
def get_latest_version(opts)
- doc = fetch_doc('https://www.haskell.org/ghc/download.html', opts)
- links = doc.css('a').to_a
- versions = links.map {|link| link.content.scan(/\A([0-9.]+)\Z/)}
- versions.find {|version| !version.empty?}[0][0]
+ tags = get_github_tags('ghc', 'ghc', opts)
+ tag = tags.find {|t| t['name'].ends_with?('-release') }['name']
+ tag[/ghc-(.*)-release/, 1]
end
end
diff --git a/lib/docs/scrapers/homebrew.rb b/lib/docs/scrapers/homebrew.rb
index 0faeaa5785..353ec01d2b 100644
--- a/lib/docs/scrapers/homebrew.rb
+++ b/lib/docs/scrapers/homebrew.rb
@@ -1,8 +1,10 @@
module Docs
class Homebrew < UrlScraper
+ # include MultipleBaseUrls
+
self.name = 'Homebrew'
self.type = 'simple'
- self.release = '4.6.15'
+ self.release = '6.0.3'
self.base_url = 'https://docs.brew.sh/'
self.links = {
home: 'https://brew.sh',
@@ -11,9 +13,29 @@ class Homebrew < UrlScraper
html_filters.push 'homebrew/entries', 'homebrew/clean_html'
- options[:container] = ->(filter) { filter.root_page? ? '#home' : '#page' }
+ options[:container] = ->(filter) do
+ if filter.current_url.path.include?('rubydoc')
+ '#content'
+ else
+ '#default'
+ end
+ end
- options[:skip_patterns] = [/maintainer/i, /core\-contributor/i, /kickstarter/i, /governance/i]
+ # Mostly handbook articles with no code.
+ options[:skip_patterns] = [
+ /maintainer/i,
+ /core\-contributor/i,
+ /kickstarter/i,
+ /governance/i,
+ /responsible/i,
+ /leadership/i,
+ /prose/i,
+ /expense/i,
+ /creating-a-homebrew-issue/i,
+ /homebrew-and-java/i,
+ /checksum_deprecation/i,
+ /Working-with/i
+ ]
options[:attribution] = <<-HTML
© 2009–present Homebrew contributors
diff --git a/lib/docs/scrapers/htmx.rb b/lib/docs/scrapers/htmx.rb
index 719277b2b9..ec1f51e07d 100644
--- a/lib/docs/scrapers/htmx.rb
+++ b/lib/docs/scrapers/htmx.rb
@@ -7,27 +7,35 @@ class Htmx < UrlScraper
home: 'https://htmx.org/',
code: 'https://github.com/bigskysoftware/htmx'
}
- self.release = '1.9.10'
- self.base_url = "https://htmx.org/"
self.initial_paths = %w(reference/)
html_filters.push 'htmx/entries', 'htmx/clean_html'
options[:trailing_slash] = true
- options[:container] = '.content'
+ options[:container] = '.content'
options[:download_images] = false
options[:skip_patterns] = [
- /\Aessays/,
- /\Aexamples/,
- /\Amigration-guide/,
- /\Aposts/,
- ]
+ /\Aessays/,
+ /\Aexamples/,
+ /\Amigration-guide/,
+ /\Aposts/,
+ ]
- # https://github.com/bigskysoftware/htmx/blob/master/LICENSE
+ # https://github.com/bigskysoftware/htmx/blob/master/LICENSE
options[:attribution] = <<-HTML
- Licensed under the Zero-Clause BSD License.
+ Licensed under the Zero-Clause BSD License.
HTML
+ version do
+ self.release = '2.0.7'
+ self.base_url = "https://htmx.org/"
+ end
+
+ version '1' do
+ self.release = '1.9.12'
+ self.base_url = "https://v1.htmx.org/"
+ end
+
def get_latest_version(opts)
get_npm_version('htmx.org', opts)
end
diff --git a/lib/docs/scrapers/http.rb b/lib/docs/scrapers/http.rb
index a2f4a7f9b0..535dc9ec2f 100644
--- a/lib/docs/scrapers/http.rb
+++ b/lib/docs/scrapers/http.rb
@@ -2,7 +2,7 @@ module Docs
class Http < Mdn
include MultipleBaseUrls
- # release = '2022-11-17'
+ # release = '2026-06-21'
self.name = 'HTTP'
self.base_urls = [
'https://developer.mozilla.org/en-US/docs/Web/HTTP',
@@ -18,14 +18,18 @@ class Http < Mdn
Licensed under the Creative Commons Attribution-ShareAlike License v2.5 or later.
HTML
- html_filters.push 'http/clean_html', 'http/entries', 'title'
+ # 'http/clean_html' must run before 'mdn/clean_html', which unwraps all
+ # /
child anchors and would otherwise discard the datatracker
+ # section-id anchors before they can be promoted to their heading.
+ html_filters.insert_before 'mdn/clean_html', 'http/clean_html'
+ html_filters.push 'http/entries', 'title'
options[:root_title] = 'HTTP'
options[:title] = ->(filter) do
filter.current_url.host == 'datatracker.ietf.org' ? false : filter.default_title
end
options[:container] = ->(filter) do
- filter.current_url.host == 'datatracker.ietf.org' ? '.content' : Docs::Mdn.options[:container]
+ filter.current_url.host == 'datatracker.ietf.org' ? '.rfcmarkup, .rfchtml' : Docs::Mdn.options[:container]
end
options[:skip_links] = ->(filter) do
filter.current_url.host == 'datatracker.ietf.org'
@@ -47,11 +51,14 @@ class Http < Mdn
def initial_urls
%w(https://developer.mozilla.org/en-US/docs/Web/HTTP
https://datatracker.ietf.org/doc/html/rfc4918
+ https://datatracker.ietf.org/doc/html/rfc6265
https://datatracker.ietf.org/doc/html/rfc9110
https://datatracker.ietf.org/doc/html/rfc9111
https://datatracker.ietf.org/doc/html/rfc9112
https://datatracker.ietf.org/doc/html/rfc9113
https://datatracker.ietf.org/doc/html/rfc9114
+ https://datatracker.ietf.org/doc/html/rfc9651
+ https://datatracker.ietf.org/doc/html/rfc9931
https://datatracker.ietf.org/doc/html/rfc5023)
end
end
diff --git a/lib/docs/scrapers/i3.rb b/lib/docs/scrapers/i3.rb
index 899a718acf..2f80785c74 100644
--- a/lib/docs/scrapers/i3.rb
+++ b/lib/docs/scrapers/i3.rb
@@ -3,7 +3,7 @@ class I3 < UrlScraper
self.name = 'i3'
self.type = 'simple'
self.slug = 'i3'
- self.release = '4.24'
+ self.release = '4.25.1'
self.base_url = 'https://i3wm.org/docs/userguide.html'
self.links = {
home: 'https://i3wm.org/',
diff --git a/lib/docs/scrapers/jq.rb b/lib/docs/scrapers/jq.rb
index 38e29ffecd..882e150ad2 100644
--- a/lib/docs/scrapers/jq.rb
+++ b/lib/docs/scrapers/jq.rb
@@ -3,12 +3,12 @@ class Jq < UrlScraper
self.name = 'jq'
self.slug = 'jq'
self.type = 'jq'
- self.release = '1.7'
+ self.release = '1.8'
self.links = {
- home: 'https://jqlang.github.io/jq/'
+ home: 'https://jqlang.org/'
}
- self.base_url = "https://jqlang.github.io/jq/manual/v#{self.release}/index.html"
+ self.base_url = "https://jqlang.org/manual/v#{self.release}/"
html_filters.push 'jq/entries', 'jq/clean_html'
diff --git a/lib/docs/scrapers/jquery/jquery_core.rb b/lib/docs/scrapers/jquery/jquery_core.rb
index 3b761e792e..9af2f32f54 100644
--- a/lib/docs/scrapers/jquery/jquery_core.rb
+++ b/lib/docs/scrapers/jquery/jquery_core.rb
@@ -1,7 +1,7 @@
module Docs
class JqueryCore < Jquery
self.name = 'jQuery'
- self.release = '3.7.1'
+ self.release = '4.0.0'
self.base_url = 'https://api.jquery.com/'
self.initial_paths = %w(/index/index)
self.links = {
diff --git a/lib/docs/scrapers/julia.rb b/lib/docs/scrapers/julia.rb
index dc8d78a984..50f0337a97 100644
--- a/lib/docs/scrapers/julia.rb
+++ b/lib/docs/scrapers/julia.rb
@@ -7,10 +7,21 @@ class Julia < UrlScraper
options[:attribution] = <<-HTML
- © 2009–2024 Jeff Bezanson, Stefan Karpinski, Viral B. Shah, and other contributors
+ © 2009–2026 Jeff Bezanson, Stefan Karpinski, Viral B. Shah, and other contributors
Licensed under the MIT License.
HTML
+ version '1.12' do
+ self.release = '1.12.6'
+ self.base_url = "https://docs.julialang.org/en/v#{version}/"
+ self.type = 'julia'
+
+ html_filters.push 'julia/entries', 'julia/clean_html'
+
+ options[:container] = '.docs-main'
+ options[:only_patterns] = [/\Amanual\//, /\Abase\//, /\Astdlib\//]
+ end
+
version '1.11' do
self.release = '1.11.2'
self.base_url = "https://docs.julialang.org/en/v#{version}/"
diff --git a/lib/docs/scrapers/lodash.rb b/lib/docs/scrapers/lodash.rb
index bce625e615..d8c9f1f143 100644
--- a/lib/docs/scrapers/lodash.rb
+++ b/lib/docs/scrapers/lodash.rb
@@ -19,7 +19,7 @@ class Lodash < UrlScraper
HTML
version '4' do
- self.release = '4.17.10'
+ self.release = '4.18.1'
self.base_url = "https://lodash.com/docs/#{release}"
end
diff --git a/lib/docs/scrapers/love.rb b/lib/docs/scrapers/love.rb
index f702da8be9..1979db059a 100644
--- a/lib/docs/scrapers/love.rb
+++ b/lib/docs/scrapers/love.rb
@@ -43,8 +43,7 @@ class Love < UrlScraper
HTML
def get_latest_version(opts)
- doc = fetch_doc('https://love2d.org/wiki/Version_History', opts)
- doc.at_css('#mw-content-text table a').content
+ get_github_tags('love2d', 'love', opts).first['name']
end
end
end
diff --git a/lib/docs/scrapers/lua.rb b/lib/docs/scrapers/lua.rb
index b7989fda83..abfdab98c5 100644
--- a/lib/docs/scrapers/lua.rb
+++ b/lib/docs/scrapers/lua.rb
@@ -11,10 +11,15 @@ class Lua < UrlScraper
options[:skip_links] = true
options[:attribution] = <<-HTML
- © 1994–2020 Lua.org, PUC-Rio.
+ © 1994–2025 Lua.org, PUC-Rio.
Licensed under the MIT License.
HTML
+ version '5.5' do
+ self.release = '5.5.0'
+ self.base_url = 'https://www.lua.org/manual/5.5/'
+ end
+
version '5.4' do
self.release = '5.4.1'
self.base_url = 'https://www.lua.org/manual/5.4/'
diff --git a/lib/docs/scrapers/maplibre_gl.rb b/lib/docs/scrapers/maplibre_gl.rb
new file mode 100644
index 0000000000..a36ca6d3d5
--- /dev/null
+++ b/lib/docs/scrapers/maplibre_gl.rb
@@ -0,0 +1,47 @@
+module Docs
+ class MaplibreGl < UrlScraper
+ self.name = 'MapLibre GL JS'
+ self.slug = 'maplibre_gl'
+ self.type = 'maplibre_gl'
+ self.release = '5.24.0'
+ self.base_url = 'https://maplibre.org/maplibre-gl-js/docs/'
+ self.root_path = '/'
+ self.links = {
+ home: 'https://maplibre.org/maplibre-gl-js/docs/',
+ code: 'https://github.com/maplibre/maplibre-gl-js'
+ }
+
+ html_filters.push 'maplibre_gl/clean_html', 'maplibre_gl/entries'
+
+ options[:container] = '.md-content__inner'
+
+ # Only scrape the TypeDoc-generated API reference and the guides.
+ # The examples (200+ interactive demos) and the embedded style
+ # specification are excluded as they aren't API reference material.
+ options[:only_patterns] = [
+ %r{\AAPI/classes/},
+ %r{\AAPI/functions/},
+ %r{\AAPI/type-aliases/},
+ %r{\AAPI/interfaces/},
+ %r{\AAPI/enumerations/},
+ %r{\AAPI/variables/},
+ %r{\Aguides/}
+ ]
+
+ # The site's navigation links point at the `www.` host, which then
+ # redirects (301) to the bare domain used by `base_url`. Rewrite them so
+ # they are recognised as internal URLs and get followed.
+ options[:fix_urls] = ->(url) do
+ url.sub(%r{\Ahttps://www\.maplibre\.org/}, 'https://maplibre.org/')
+ end
+
+ options[:attribution] = <<-HTML
+ © MapLibre contributors
+ Licensed under the 3-Clause BSD License.
+ HTML
+
+ def get_latest_version(opts)
+ get_npm_version('maplibre-gl', opts)
+ end
+ end
+end
diff --git a/lib/docs/scrapers/matplotlib.rb b/lib/docs/scrapers/matplotlib.rb
index 5e30998c2b..5e2447a31e 100644
--- a/lib/docs/scrapers/matplotlib.rb
+++ b/lib/docs/scrapers/matplotlib.rb
@@ -16,12 +16,12 @@ class Matplotlib < UrlScraper
options[:skip] = %w(api_changes.html tutorial.html faq.html)
options[:attribution] = <<-HTML
- © 2012–2023 Matplotlib Development Team. All rights reserved.
+ © 2012–2026 Matplotlib Development Team. All rights reserved.
Licensed under the Matplotlib License Agreement.
HTML
version do
- self.release = '3.9.2'
+ self.release = '3.10.8'
self.base_urls = [
"https://matplotlib.org/stable/api/",
"https://matplotlib.org/stable/mpl_toolkits/mplot3d/",
@@ -29,6 +29,15 @@ class Matplotlib < UrlScraper
]
end
+ version '3.9' do
+ self.release = '3.9.3'
+ self.base_urls = [
+ "https://matplotlib.org/#{release}/api/",
+ "https://matplotlib.org/#{release}/mpl_toolkits/mplot3d/",
+ "https://matplotlib.org/#{release}/mpl_toolkits/axes_grid/api/"
+ ]
+ end
+
version '3.8' do
self.release = '3.8.4'
self.base_urls = [
diff --git a/lib/docs/scrapers/mdn/html.rb b/lib/docs/scrapers/mdn/html.rb
index 9a1f3ab4e6..319b73cbf6 100644
--- a/lib/docs/scrapers/mdn/html.rb
+++ b/lib/docs/scrapers/mdn/html.rb
@@ -2,7 +2,7 @@ module Docs
class Html < Mdn
prepend FixInternalUrlsBehavior
- # release = '2025-09-15'
+ # release = '2026-05-26'
self.name = 'HTML'
self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/HTML'
self.links = {
diff --git a/lib/docs/scrapers/mdn/javascript.rb b/lib/docs/scrapers/mdn/javascript.rb
index b8d9391151..913da9a418 100644
--- a/lib/docs/scrapers/mdn/javascript.rb
+++ b/lib/docs/scrapers/mdn/javascript.rb
@@ -3,7 +3,7 @@ class Javascript < Mdn
prepend FixInternalUrlsBehavior
prepend FixRedirectionsBehavior
- # release = '2025-09-15'
+ # release = '2026-07-08'
self.name = 'JavaScript'
self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference'
self.links = {
diff --git a/lib/docs/scrapers/mongoose.rb b/lib/docs/scrapers/mongoose.rb
index bcd9673329..1818c78c59 100644
--- a/lib/docs/scrapers/mongoose.rb
+++ b/lib/docs/scrapers/mongoose.rb
@@ -29,9 +29,7 @@ class Mongoose < UrlScraper
HTML
def get_latest_version(opts)
- doc = fetch_doc('https://mongoosejs.com/docs/', opts)
- label = doc.at_css('.pure-menu-link').content.strip
- label.sub(/Version /, '')
+ get_github_tags('Automattic', 'mongoose', opts).find {|t| t['name'].starts_with?(/\d/)}['name']
end
end
end
diff --git a/lib/docs/scrapers/nginx.rb b/lib/docs/scrapers/nginx.rb
index e7af592414..d7bdf11131 100644
--- a/lib/docs/scrapers/nginx.rb
+++ b/lib/docs/scrapers/nginx.rb
@@ -2,7 +2,7 @@ module Docs
class Nginx < UrlScraper
self.name = 'nginx'
self.type = 'nginx'
- self.release = '1.29.0'
+ self.release = '1.31.1'
self.base_url = 'https://nginx.org/en/docs/'
self.links = {
home: 'https://nginx.org/',
@@ -23,7 +23,7 @@ class Nginx < UrlScraper
# http://nginx.org/LICENSE
options[:attribution] = <<-HTML
© 2002-2021 Igor Sysoev
- © 2011-2025 Nginx, Inc.
+ © 2011-2026 Nginx, Inc.
Licensed under the BSD License.
HTML
diff --git a/lib/docs/scrapers/nix.rb b/lib/docs/scrapers/nix.rb
index b2ee5c8798..31deb9141b 100644
--- a/lib/docs/scrapers/nix.rb
+++ b/lib/docs/scrapers/nix.rb
@@ -23,7 +23,7 @@ class Nix < UrlScraper
def get_latest_version(opts)
doc = fetch_doc('https://nixos.org/manual/nix/stable/', opts)
- doc.at_css('a.active')['href'].scan(/([0-9.]+)/)[0][0]
+ doc.at_css('h1.menu-title').content.scan(/([0-9.]+)/).first.first
end
end
end
diff --git a/lib/docs/scrapers/node.rb b/lib/docs/scrapers/node.rb
index 7331de7d59..82eed214c3 100644
--- a/lib/docs/scrapers/node.rb
+++ b/lib/docs/scrapers/node.rb
@@ -24,12 +24,12 @@ class Node < UrlScraper
HTML
version do
- self.release = '25.0.0'
+ self.release = '26.3.1'
self.base_url = 'https://nodejs.org/api/'
end
version '24 LTS' do
- self.release = '24.10.0'
+ self.release = '24.14.0'
self.base_url = 'https://nodejs.org/dist/latest-v24.x/docs/api/'
end
diff --git a/lib/docs/scrapers/numpy.rb b/lib/docs/scrapers/numpy.rb
index a679caaae2..61c07cb66b 100644
--- a/lib/docs/scrapers/numpy.rb
+++ b/lib/docs/scrapers/numpy.rb
@@ -1,7 +1,7 @@
module Docs
# Requires downloading the documents to local disk first.
# Go to https://numpy.org/doc/, click "HTML+zip" to download
- # (example url: https://numpy.org/doc/2.2/numpy-html.zip),
+ # (example url: https://numpy.org/doc/2.4/numpy-html.zip),
# then extract into "docs/numpy~#{version}/"
class Numpy < FileScraper
self.name = 'NumPy'
@@ -30,6 +30,12 @@ class Numpy < FileScraper
Licensed under the 3-clause BSD License.
HTML
+ version '2.4' do
+ self.release = '2.4'
+ self.base_url = "https://numpy.org/doc/#{self.version}/"
+ options[:container] = nil
+ end
+
version '2.2' do
self.release = '2.2'
self.base_url = "https://numpy.org/doc/#{self.version}/"
diff --git a/lib/docs/scrapers/nushell.rb b/lib/docs/scrapers/nushell.rb
index c3e11e2075..6e7ef8adfb 100644
--- a/lib/docs/scrapers/nushell.rb
+++ b/lib/docs/scrapers/nushell.rb
@@ -6,7 +6,7 @@ class Nushell < UrlScraper
self.name = "Nushell"
self.slug = "nushell"
self.type = "nushell"
- self.release = "0.85.0"
+ self.release = "0.113.1"
self.links = {
home: "https://www.nushell.sh/",
code: "https://github.com/nushell/nushell",
@@ -14,11 +14,11 @@ class Nushell < UrlScraper
html_filters.push "nushell/clean_html", "nushell/entries", "nushell/fix_links"
- options[:container] = '.theme-container'
+ options[:container] = 'main > [vp-content]'
options[:follow_links] = true
options[:title] = "Nushell"
options[:attribution] = <<-HTML
- Copyright © 2019–2023 The Nushell Project Developers
+ Copyright © 2019–2026 The Nushell Project Developers
Licensed under the MIT License.
HTML
diff --git a/lib/docs/scrapers/ocaml.rb b/lib/docs/scrapers/ocaml.rb
index 97a25a9413..10ba166c23 100644
--- a/lib/docs/scrapers/ocaml.rb
+++ b/lib/docs/scrapers/ocaml.rb
@@ -21,11 +21,11 @@ class Ocaml < UrlScraper
]
options[:attribution] = <<-HTML
- © 1995-2024 INRIA.
+ © 1995-2025 INRIA.
HTML
version '' do
- self.release = '5.3'
+ self.release = '5.4'
self.base_url = "https://ocaml.org/manual/#{self.release}/"
end
diff --git a/lib/docs/scrapers/octave.rb b/lib/docs/scrapers/octave.rb
index ee798d58dc..83a9bad8c2 100644
--- a/lib/docs/scrapers/octave.rb
+++ b/lib/docs/scrapers/octave.rb
@@ -10,6 +10,8 @@ class Octave < UrlScraper
html_filters.push 'octave/clean_html', 'octave/entries', 'title'
+ options[:container] = 'body > div[id], body > .contents'
+
options[:skip] = %w(
Copying.html
Preface.html
@@ -23,19 +25,24 @@ class Octave < UrlScraper
options[:root_title] = 'GNU Octave'
options[:attribution] = <<-HTML
- © 1996–2025 The Octave Project Developers
+ © 1996–2026 The Octave Project Developers
Permission is granted to make and distribute verbatim copies of this manual provided the copyright notice and this permission notice are preserved on all copies.
Permission is granted to copy and distribute modified versions of this manual under the conditions for verbatim copying, provided that the entire resulting derived work is distributed under the terms of a permission notice identical to this one.
Permission is granted to copy and distribute translations of this manual into another language, under the above conditions for modified versions.
HTML
+ version '11' do
+ self.release = '11.1.0'
+ self.base_url = "https://docs.octave.org/v#{self.release}/"
+ end
+
version '10' do
- self.release = '10.1.0'
+ self.release = '10.3.0'
self.base_url = "https://docs.octave.org/v#{self.release}/"
end
version '9' do
- self.release = '9.2.0'
+ self.release = '9.4.0'
self.base_url = "https://docs.octave.org/v#{self.release}/"
end
diff --git a/lib/docs/scrapers/odin.rb b/lib/docs/scrapers/odin.rb
new file mode 100644
index 0000000000..e54a43d490
--- /dev/null
+++ b/lib/docs/scrapers/odin.rb
@@ -0,0 +1,62 @@
+module Docs
+ class Odin < UrlScraper
+ include MultipleBaseUrls
+ self.name = 'Odin'
+ self.slug = 'odin'
+ self.type = 'odin'
+ self.release = '2026-07'
+ self.base_urls = [
+ 'https://odin-lang.org/',
+ 'https://pkg.odin-lang.org/'
+ ]
+
+ def initial_urls
+ [
+ 'https://odin-lang.org/docs',
+ 'https://odin-lang.org/spec',
+ 'https://pkg.odin-lang.org/base',
+ 'https://pkg.odin-lang.org/core',
+ 'https://pkg.odin-lang.org/vendor'
+ ]
+ end
+
+ self.root_path = 'https://odin-lang.org/'
+ self.initial_paths = %w(base core vendor)
+
+ self.links = {
+ home: 'https://odin-lang.org/',
+ code: 'https://github.com/odin-lang/Odin'
+ }
+
+ html_filters.push 'odin/entries', 'odin/clean_html'
+ options[:download_images] = false
+
+ options[:container] = '.odin-main'
+
+ options[:only_patterns] = [
+ /docs/,
+ /spec/,
+ /base/,
+ /core/,
+ /vendor/
+ ]
+ options[:trailing_slash] = false
+ options[:skip] = %w(
+ docs/examples
+ docs/nightly
+ docs/odin-book
+ docs/spec
+ docs/packages
+ )
+
+ options[:attribution] = <<-HTML
+ © 2016-#{Date.today.year} Ginger Bill
+ Licensed under the zlib License.
+ HTML
+
+ def get_latest_version(opts)
+ get_latest_github_release('odin-lang', 'Odin', opts)[4..]
+ end
+
+ end
+end
diff --git a/lib/docs/scrapers/openlayers.rb b/lib/docs/scrapers/openlayers.rb
index b9bf9d347a..3d08fae679 100644
--- a/lib/docs/scrapers/openlayers.rb
+++ b/lib/docs/scrapers/openlayers.rb
@@ -3,7 +3,7 @@ class Openlayers < UrlScraper
self.name = 'OpenLayers'
self.type = 'openlayers'
self.slug = 'openlayers'
- self.release = '10.6.1'
+ self.release = '10.8.0'
self.base_url = "https://openlayers.org/en/latest/apidoc/"
self.links = {
home: 'https://openlayers.org/',
diff --git a/lib/docs/scrapers/opentofu.rb b/lib/docs/scrapers/opentofu.rb
new file mode 100644
index 0000000000..52b2757157
--- /dev/null
+++ b/lib/docs/scrapers/opentofu.rb
@@ -0,0 +1,47 @@
+module Docs
+ class Opentofu < UrlScraper
+ self.name = 'OpenTofu'
+ self.type = 'opentofu'
+ self.links = {
+ home: 'https://opentofu.org/',
+ }
+
+ html_filters.push 'opentofu/entries', 'opentofu/clean_html'
+
+ # Empty spans are used by Prism for code highlighting.
+ # Don't clean them
+ options[:clean_text] = false
+ options[:trailing_slash] = true
+ # https://github.com/opentofu/opentofu/blob/main/LICENSE
+ options[:attribution] = <<-HTML
+ Copyright (c) The OpenTofu Authors
+ Copyright (c) 2014 HashiCorp, Inc.
+ Mozilla Public License, version 2.0
+ HTML
+
+ def get_latest_version(opts)
+ contents = get_latest_github_release('opentofu', 'opentofu', opts)
+ contents.sub("v", "")
+ end
+
+ version '1.12' do
+ self.release = '1.12.0'
+ self.base_url = "https://opentofu.org/docs/v#{self.version}/"
+ end
+
+ version '1.11' do
+ self.release = '1.11.5'
+ self.base_url = "https://opentofu.org/docs/v#{self.version}/"
+ end
+
+ version '1.10' do
+ self.release = '1.10.9'
+ self.base_url = "https://opentofu.org/docs/v#{self.version}/"
+ end
+
+ version '1.9' do
+ self.release = '1.9.4'
+ self.base_url = "https://opentofu.org/docs/v#{self.version}/"
+ end
+ end
+end
diff --git a/lib/docs/scrapers/pandas.rb b/lib/docs/scrapers/pandas.rb
index 872930d31c..a9c521bf97 100644
--- a/lib/docs/scrapers/pandas.rb
+++ b/lib/docs/scrapers/pandas.rb
@@ -18,6 +18,29 @@ class Pandas < FileScraper
Licensed under the 3-clause BSD License.
HTML
+ version '3' do
+ self.release = '3.0.8'
+ self.base_url = "https://pandas.pydata.org/pandas-docs/version/#{self.release}/"
+
+ html_filters.push 'pandas/clean_html', 'pandas/entries'
+
+ options[:container] = 'main section'
+
+ options[:skip_patterns] = [
+ /development/,
+ /getting_started/,
+ /whatsnew/
+ ]
+
+ options[:skip] = [
+ 'panel.html',
+ 'pandas.pdf',
+ 'pandas.zip',
+ 'ecosystem.html'
+ ]
+
+ end
+
version '2' do
self.release = '2.3.0'
self.base_url = "https://pandas.pydata.org/pandas-docs/version/#{self.release}/"
diff --git a/lib/docs/scrapers/phoenix.rb b/lib/docs/scrapers/phoenix.rb
index d115ef053e..7e7018c4c6 100644
--- a/lib/docs/scrapers/phoenix.rb
+++ b/lib/docs/scrapers/phoenix.rb
@@ -1,34 +1,44 @@
module Docs
class Phoenix < UrlScraper
+ include MultipleBaseUrls
+
self.type = 'elixir'
- self.release = '1.6.11'
- self.base_url = 'https://hexdocs.pm/'
- self.root_path = 'phoenix/Phoenix.html'
- self.initial_paths = %w(
- phoenix/api-reference.html
- ecto/api-reference.html
- phoenix_html/api-reference.html
- phoenix_live_view/api-reference.html
- phoenix_pubsub/api-reference.html
- plug/api-reference.html)
+ self.release = '1.8.8'
+ self.root_path = 'overview.html'
self.links = {
home: 'http://www.phoenixframework.org',
code: 'https://github.com/phoenixframework/phoenix'
}
+ self.base_urls = %w(
+ https://phoenix.hexdocs.pm/
+ https://ecto.hexdocs.pm/
+ https://phoenix-html.hexdocs.pm/
+ https://phoenix-live-view.hexdocs.pm/
+ https://phoenix-pubsub.hexdocs.pm/
+ https://plug.hexdocs.pm/
+ )
+
+ def initial_urls
+ %w(
+ https://phoenix.hexdocs.pm/overview.html
+ https://ecto.hexdocs.pm/Ecto.html
+ https://phoenix-html.hexdocs.pm/Phoenix.HTML.html
+ https://phoenix-live-view.hexdocs.pm/welcome.html
+ https://phoenix-pubsub.hexdocs.pm/Phoenix.PubSub.html
+ https://plug.hexdocs.pm/readme.html
+ )
+ end
+
html_filters.push 'elixir/clean_html', 'elixir/entries'
options[:container] = '#content'
- options[:skip_patterns] = [/extra-api-reference/, /js/, /\d+\.\d+\.\d+/]
- options[:only_patterns] = [
- /\Aphoenix\//,
- /\Aecto\//,
- /\Aphoenix_pubsub\//,
- /\Aphoenix_html\//,
- /\Aphoenix_live_view\//,
- /\Aplug\//
- ]
+ # Filter docs for JS libraries, these use a different HTML layout.
+ # e.g. https://phoenix.hexdocs.pm/js/ and https://phoenix-live-view.hexdocs.pm/1.2.0/js/
+ #
+ # Only match on `js` directories so we still catch normal pages like https://phoenix-live-view.hexdocs.pm/js-interop.html
+ options[:skip_patterns] = [%r{(\A|/)js/}]
options[:attribution] = -> (filter) {
if filter.slug.start_with?('ecto')
@@ -56,7 +66,7 @@ class Phoenix < UrlScraper
}
def get_latest_version(opts)
- doc = fetch_doc('https://hexdocs.pm/phoenix/Phoenix.html', opts)
+ doc = fetch_doc('https://phoenix.hexdocs.pm/api-reference.html', opts)
doc.at_css('.sidebar-projectVersion').content.strip[1..-1]
end
end
diff --git a/lib/docs/scrapers/php.rb b/lib/docs/scrapers/php.rb
index 5a2f42e24c..898110da2e 100644
--- a/lib/docs/scrapers/php.rb
+++ b/lib/docs/scrapers/php.rb
@@ -5,7 +5,7 @@ class Php < FileScraper
self.name = 'PHP'
self.type = 'php'
- self.release = '8.4'
+ self.release = '8.5'
self.base_url = 'https://www.php.net/manual/en/'
self.root_path = 'index.html'
self.initial_paths = %w(
@@ -62,7 +62,7 @@ class Php < FileScraper
options[:skip_patterns] = [/mysqlnd/, /xdevapi/i]
options[:attribution] = <<-HTML
- © 1997–2024 The PHP Documentation Group
+ © 1997–2025 The PHP Documentation Group
Licensed under the Creative Commons Attribution License v3.0 or later.
HTML
diff --git a/lib/docs/scrapers/phpunit.rb b/lib/docs/scrapers/phpunit.rb
index 12efbbfcba..ba1dafc5bd 100644
--- a/lib/docs/scrapers/phpunit.rb
+++ b/lib/docs/scrapers/phpunit.rb
@@ -17,14 +17,23 @@ class Phpunit < UrlScraper
options[:title] = false
options[:attribution] = <<-HTML
- © 2005–2025 Sebastian Bergmann
+ © 2005–2026 Sebastian Bergmann
Licensed under the Creative Commons Attribution 3.0 Unported License.
HTML
FILTERS = %w(phpunit/clean_html phpunit/entries title)
version do
- self.release = '12.0'
+ self.release = '13.2'
+ self.base_url = "https://docs.phpunit.de/en/#{release}/"
+
+ html_filters.push FILTERS
+
+ options[:container] = '.document'
+ end
+
+ version '12' do
+ self.release = '12.5'
self.base_url = "https://docs.phpunit.de/en/#{release}/"
html_filters.push FILTERS
diff --git a/lib/docs/scrapers/playwright.rb b/lib/docs/scrapers/playwright.rb
index 479d7476e4..0694722374 100644
--- a/lib/docs/scrapers/playwright.rb
+++ b/lib/docs/scrapers/playwright.rb
@@ -2,7 +2,7 @@ module Docs
class Playwright < UrlScraper
self.name = 'Playwright'
self.type = 'simple'
- self.release = '1.55.0'
+ self.release = '1.61.0'
self.base_url = 'https://playwright.dev/docs/'
self.root_path = 'intro'
self.links = {
@@ -16,7 +16,7 @@ class Playwright < UrlScraper
# https://github.com/microsoft/playwright/blob/main/LICENSE
options[:attribution] = <<-HTML
- © 2025 Microsoft
+ © 2026 Microsoft
Licensed under the Apache License, Version 2.0.
HTML
diff --git a/lib/docs/scrapers/polars.rb b/lib/docs/scrapers/polars.rb
new file mode 100644
index 0000000000..3ebcd2557a
--- /dev/null
+++ b/lib/docs/scrapers/polars.rb
@@ -0,0 +1,58 @@
+module Docs
+ class Polars < UrlScraper
+ self.name = 'Polars'
+ self.type = 'sphinx'
+ self.release = '1.41.0'
+ self.base_url = 'https://docs.pola.rs/api/python/stable/reference/'
+ self.root_path = 'index.html'
+ self.links = {
+ home: 'https://pola.rs/',
+ code: 'https://github.com/pola-rs/polars'
+ }
+
+ html_filters.push 'polars/entries', 'sphinx/clean_html', 'polars/clean_html'
+
+ # pydata-sphinx-theme keeps the page content in the article body.
+ options[:container] = 'article.bd-article'
+
+ options[:skip_patterns] = [/_changelog/, /whatsnew/]
+
+ # https://github.com/pola-rs/polars/blob/main/LICENSE
+ options[:attribution] = <<-HTML
+ © 2020 Ritchie Vink
+ © 2022 Polars contributors
+ Licensed under the MIT License.
+ HTML
+
+ # Polars tags both Rust (rs-*) and Python (py-*) releases in the same repo.
+ # The tags API only lists recent Rust ones, but the latest GitHub release is
+ # always the Python one, so use that and drop the py- prefix.
+ def get_latest_version(opts)
+ get_latest_github_release('pola-rs', 'polars', opts).sub(/\Apy-/, '')
+ end
+
+ private
+
+ def parse(response)
+ if response.body.include?('class="sig')
+ doc = Nokogiri::HTML5(response.body)
+ doc.css('.sig').each do |node|
+ node.css('.headerlink').remove
+ node.css('.reference.external').each { |a| a.remove if a.text.strip == '[source]' }
+ sig = node.text.gsub(/\s+/, ' ').strip
+ if (m = sig.match(/\A(.+?\()\s*(.+?)\s*(\).*)\z/m))
+ head, params, tail = m[1], m[2], m[3]
+ split_params = params.split(/,\s+/).map { |p| p.sub(/,\z/, '') }.reject(&:empty?)
+ sig = "#{head}\n #{split_params.join(",\n ")},\n#{tail}" unless split_params.empty?
+ end
+ pre = Nokogiri::XML::Node.new('pre', doc)
+ pre['data-language'] = 'python'
+ pre.content = sig
+ node.replace(pre)
+ end
+ response.body.replace(doc.to_html)
+ end
+ super
+ end
+ end
+end
diff --git a/lib/docs/scrapers/postgresql.rb b/lib/docs/scrapers/postgresql.rb
index 38ac56a06e..3d56492cd9 100644
--- a/lib/docs/scrapers/postgresql.rb
+++ b/lib/docs/scrapers/postgresql.rb
@@ -52,12 +52,12 @@ class Postgresql < UrlScraper
/\Aunsupported-features/ ]
options[:attribution] = <<-HTML
- © 1996–2025 The PostgreSQL Global Development Group
+ © 1996–2026 The PostgreSQL Global Development Group
Licensed under the PostgreSQL License.
HTML
version '18' do
- self.release = '18.0'
+ self.release = '18.3'
self.base_url = "https://www.postgresql.org/docs/#{version}/"
end
@@ -70,7 +70,7 @@ class Postgresql < UrlScraper
self.release = '16.1'
self.base_url = "https://www.postgresql.org/docs/#{version}/"
end
-
+
version '15' do
self.release = '15.4'
self.base_url = "https://www.postgresql.org/docs/#{version}/"
diff --git a/lib/docs/scrapers/powershell.rb b/lib/docs/scrapers/powershell.rb
new file mode 100644
index 0000000000..046ad405da
--- /dev/null
+++ b/lib/docs/scrapers/powershell.rb
@@ -0,0 +1,86 @@
+module Docs
+ class Powershell < FileScraper
+ self.name = 'PowerShell'
+ self.type = 'simple'
+ self.root_path = 'Microsoft.PowerShell.Core/Get-Help.html'
+ self.links = {
+ home: 'https://learn.microsoft.com/powershell',
+ code: 'https://github.com/MicrosoftDocs/PowerShell-Docs'
+ }
+ html_filters.push 'powershell/clean_html', 'powershell/entries'
+ text_filters.replace 'attribution', 'powershell/attribution'
+
+ # https://github.com/MicrosoftDocs/PowerShell-Docs/blob/main/LICENSE-CODE.md
+ # https://github.com/MicrosoftDocs/PowerShell-Docs/blob/main/LICENSE.md
+ options[:attribution] = <<-HTML
+ © Microsoft Corporation
+ Code licensed under an MIT-style License. Documentation licensed under CC BY 4.0.
+ HTML
+
+ version '7.7' do
+ self.release = '7.7'
+ end
+
+ version '7.6' do
+ self.release = '7.6'
+ end
+
+ version '7.5' do
+ self.release = '7.5'
+ end
+
+ version '7.4' do
+ self.release = '7.4'
+ end
+
+ version '5.1' do
+ self.release = '5.1'
+ end
+
+ version 'Scripting' do
+ self.root_path = 'discover-powershell.html'
+
+ def source_directory
+ @source_directory ||= File.join(Docs::FileScraper::SOURCE_DIRECTORY, 'powershell', 'docs-conceptual')
+ end
+ end
+
+ def source_directory
+ @source_directory ||= File.join(Docs::FileScraper::SOURCE_DIRECTORY, 'powershell', self.class.version)
+ end
+
+ def get_latest_version(opts)
+ # Stable PowerShell releases are tagged like "v7.5.2"; the docs are
+ # versioned by major.minor (e.g. "7.5"), so drop the patch component.
+ get_latest_github_release('PowerShell', 'PowerShell', opts).split('.')[0..1].join('.')
+ end
+
+ # No index page, enumerate all HTML files
+ def request_all(urls)
+ assert_source_directory_exists
+ Dir.glob(File.join(source_directory, '**', '*.md')).sort.each do |path|
+ url = File.join(base_url.to_s, path.sub("#{source_directory}/", ''))
+ yield request_one(url)
+ end
+ end
+
+ private
+
+ def parse(response)
+ body = response.body.sub(/\A---\s*\n.*?\n---\s*\n/m, '')
+ html = markdown_renderer.render(body)
+ [Parser.new("#{html}").html, ""]
+ end
+
+ def markdown_renderer
+ require 'redcarpet'
+ @markdown_renderer ||= Redcarpet::Markdown.new(
+ Redcarpet::Render::HTML.new(with_toc_data: true),
+ autolink: true,
+ fenced_code_blocks: true,
+ no_intra_emphasis: true,
+ tables: true
+ )
+ end
+ end
+end
diff --git a/lib/docs/scrapers/prettier.rb b/lib/docs/scrapers/prettier.rb
index 888a042582..80bf39022c 100644
--- a/lib/docs/scrapers/prettier.rb
+++ b/lib/docs/scrapers/prettier.rb
@@ -2,7 +2,7 @@ module Docs
class Prettier < UrlScraper
self.name = 'Prettier'
self.type = 'simple'
- self.release = '3.6.2'
+ self.release = '3.8.2'
self.base_url = 'https://prettier.io/docs/'
self.links = {
home: 'https://prettier.io/',
diff --git a/lib/docs/scrapers/python.rb b/lib/docs/scrapers/python.rb
index e9edbec572..a0bd1ee480 100644
--- a/lib/docs/scrapers/python.rb
+++ b/lib/docs/scrapers/python.rb
@@ -15,6 +15,7 @@ class Python < UrlScraper
options[:skip_patterns] = [/whatsnew/]
options[:skip] = %w(
+ archives/
library/2to3.html
library/formatter.html
library/intro.html
@@ -23,12 +24,12 @@ class Python < UrlScraper
library/sunau.html)
options[:attribution] = <<-HTML
- © 2001–2025 Python Software Foundation
+ © 2001 Python Software Foundation
Licensed under the PSF License.
HTML
version '3.14' do
- self.release = '3.14.0'
+ self.release = '3.14.6'
self.base_url = "https://docs.python.org/#{self.version}/"
html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
diff --git a/lib/docs/scrapers/pytorch.rb b/lib/docs/scrapers/pytorch.rb
index cfa1d51010..e45ac3b096 100644
--- a/lib/docs/scrapers/pytorch.rb
+++ b/lib/docs/scrapers/pytorch.rb
@@ -19,6 +19,16 @@ class Pytorch < UrlScraper
PyTorch has a BSD-style license, as found in the LICENSE file.
HTML
+ version '2.9' do
+ self.release = '2.9'
+ self.base_url = "https://docs.pytorch.org/docs/#{release}/"
+ end
+
+ version '2.8' do
+ self.release = '2.8'
+ self.base_url = "https://docs.pytorch.org/docs/#{release}/"
+ end
+
version '2.7' do
self.release = '2.7'
self.base_url = "https://docs.pytorch.org/docs/#{release}/"
diff --git a/lib/docs/scrapers/qunit.rb b/lib/docs/scrapers/qunit.rb
index 125605e373..f1eed67582 100644
--- a/lib/docs/scrapers/qunit.rb
+++ b/lib/docs/scrapers/qunit.rb
@@ -4,7 +4,7 @@ module Docs
class Qunit < UrlScraper
self.name = 'QUnit'
self.type = 'qunit'
- self.release = '2.24.1'
+ self.release = '2.25.0'
self.base_url = 'https://qunitjs.com/api/'
self.root_path = '/'
self.links = {
diff --git a/lib/docs/scrapers/rabbit_mq.rb b/lib/docs/scrapers/rabbit_mq.rb
new file mode 100644
index 0000000000..9ce23eca3b
--- /dev/null
+++ b/lib/docs/scrapers/rabbit_mq.rb
@@ -0,0 +1,29 @@
+module Docs
+ class RabbitMq < UrlScraper
+ self.name = 'RabbitMQ'
+ self.type = 'rabbit_mq'
+ self.links = {
+ home: 'https://www.rabbitmq.com/',
+ code: 'https://github.com/rabbitmq/rabbitmq-server'
+ }
+
+ html_filters.push 'rabbit_mq/entries', 'rabbit_mq/clean_html'
+
+ options[:root_path] = '/docs'
+ options[:skip_patterns] = [/\/next|\/\d\.\d*/, /rabbitmq\//]
+ options[:attribution] = <<-HTML
+ Copyright © 2005-2026 Broadcom. All Rights Reserved. The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
+ Licensed under the Apache License, Version 2.0.
+ HTML
+
+ version do
+ self.base_url = 'https://www.rabbitmq.com/docs'
+ self.release = '4.3.2'
+ end
+
+ def get_latest_version(opts)
+ tags = get_github_tags("rabbitmq", "rabbitmq-server", opts)
+ tags[0]['name'][1..-1]
+ end
+ end
+end
diff --git a/lib/docs/scrapers/rdoc/minitest.rb b/lib/docs/scrapers/rdoc/minitest.rb
index a502343590..7385fb4006 100644
--- a/lib/docs/scrapers/rdoc/minitest.rb
+++ b/lib/docs/scrapers/rdoc/minitest.rb
@@ -3,11 +3,11 @@ class Minitest < Rdoc
self.name = 'Ruby / Minitest'
self.slug = 'minitest'
- self.release = '5.25.4'
+ self.release = '6.0.1'
self.links = {
+ home: 'https://minite.st',
code: 'https://github.com/minitest/minitest'
}
- self.base_url = 'https://docs.seattlerb.org/minitest/'
html_filters.replace 'rdoc/entries', 'minitest/entries'
diff --git a/lib/docs/scrapers/rdoc/rails.rb b/lib/docs/scrapers/rdoc/rails.rb
index 03aad4a356..a78cbd5042 100644
--- a/lib/docs/scrapers/rdoc/rails.rb
+++ b/lib/docs/scrapers/rdoc/rails.rb
@@ -75,6 +75,10 @@ class Rails < Rdoc
end
end
+ version '8.1' do
+ self.release = '8.1.2'
+ end
+
version '8.0' do
self.release = '8.0.1'
end
diff --git a/lib/docs/scrapers/rdoc/ruby.rb b/lib/docs/scrapers/rdoc/ruby.rb
index baba8359a1..7f7f9ac612 100644
--- a/lib/docs/scrapers/rdoc/ruby.rb
+++ b/lib/docs/scrapers/rdoc/ruby.rb
@@ -27,7 +27,6 @@ class Ruby < Rdoc
syntax_rdoc.html
extension_rdoc.html
extension_ja_rdoc.html
- Data.html
English.html
Fcntl.html
Kconv.html
@@ -69,6 +68,11 @@ class Ruby < Rdoc
Licensed under their own licenses.
HTML
+ version '4.0' do
+ self.release = '4.0.1'
+ self.root_path = 'index.html'
+ end
+
version '3.4' do
self.release = '3.4.7'
end
diff --git a/lib/docs/scrapers/redis.rb b/lib/docs/scrapers/redis.rb
index 9c8e03b19b..9c7cc12333 100644
--- a/lib/docs/scrapers/redis.rb
+++ b/lib/docs/scrapers/redis.rb
@@ -21,8 +21,7 @@ class Redis < UrlScraper
HTML
def get_latest_version(opts)
- body = fetch('http://download.redis.io/redis-stable/00-RELEASENOTES', opts)
- body.scan(/Redis Community Edition ([0-9.]+)/)[0][0]
+ get_latest_github_release('redis', 'redis', opts)
end
private
diff --git a/lib/docs/scrapers/requests.rb b/lib/docs/scrapers/requests.rb
index d031e3752a..41107d3ddf 100644
--- a/lib/docs/scrapers/requests.rb
+++ b/lib/docs/scrapers/requests.rb
@@ -6,7 +6,7 @@ class Requests < UrlScraper
home: 'https://requests.readthedocs.io/',
code: 'https://github.com/psf/requests'
}
- self.release = '2.28.1'
+ self.release = '2.32.5'
self.base_url = "https://requests.readthedocs.io/en/latest/api/"
html_filters.push 'requests/entries', 'sphinx/clean_html'
@@ -14,7 +14,7 @@ class Requests < UrlScraper
options[:container] = '.body > section'
options[:attribution] = <<-HTML
- © 2011-2022 Kenneth Reitz and other contributors
+ © 2011-2025 Kenneth Reitz and other contributors
Licensed under the Apache license.
HTML
diff --git a/lib/docs/scrapers/rust.rb b/lib/docs/scrapers/rust.rb
index 2ad145d372..3258a7a2de 100644
--- a/lib/docs/scrapers/rust.rb
+++ b/lib/docs/scrapers/rust.rb
@@ -3,7 +3,7 @@
module Docs
class Rust < UrlScraper
self.type = 'rust'
- self.release = '1.90.0'
+ self.release = '1.97.0'
self.base_url = 'https://doc.rust-lang.org/'
self.root_path = 'book/index.html'
self.initial_paths = %w(
diff --git a/lib/docs/scrapers/sqlite.rb b/lib/docs/scrapers/sqlite.rb
index c7a4cd36df..07d3daf33f 100644
--- a/lib/docs/scrapers/sqlite.rb
+++ b/lib/docs/scrapers/sqlite.rb
@@ -2,7 +2,7 @@ module Docs
class Sqlite < UrlScraper
self.name = 'SQLite'
self.type = 'sqlite'
- self.release = '3.50.0'
+ self.release = '3.53.0'
self.base_url = 'https://sqlite.org/'
self.root_path = 'docs.html'
self.initial_paths = %w(keyword_index.html)
diff --git a/lib/docs/scrapers/support_tables.rb b/lib/docs/scrapers/support_tables.rb
index 09e7eaf9e4..fc18c11cce 100644
--- a/lib/docs/scrapers/support_tables.rb
+++ b/lib/docs/scrapers/support_tables.rb
@@ -7,9 +7,12 @@ class SupportTables < Scraper
self.name = 'Support Tables'
self.slug = 'browser_support_tables'
self.type = 'support_tables'
- self.release = '1.0.30001751'
+ self.release = '1.0.30001805'
self.base_url = 'https://github.com/Fyrd/caniuse/raw/main/'
+ # Clone of https://github.com/Fyrd/caniuse into docs/caniuse
+ CANIUSE_DIRECTORY = 'docs/caniuse'
+
# https://github.com/Fyrd/caniuse/blob/main/LICENSE
options[:attribution] = <<-HTML
© 2020 Alexis Deveria
@@ -17,13 +20,16 @@ class SupportTables < Scraper
HTML
def build_pages
- url = 'https://github.com/Fyrd/caniuse/raw/main/data.json'
- instrument 'running.scraper', urls: [url]
+ if Dir.exist?(CANIUSE_DIRECTORY)
+ system('git', '-C', CANIUSE_DIRECTORY, 'pull')
+ else
+ system('git', 'clone', '--depth', '32', 'https://github.com/Fyrd/caniuse', CANIUSE_DIRECTORY)
+ end
- response = Request.run(url)
- instrument 'process_response.scraper', response: response
+ path = File.join(CANIUSE_DIRECTORY, 'data.json')
+ instrument 'running.scraper', urls: [path]
- data = JSON.parse(response.body)
+ data = JSON.parse(File.read(path))
instrument 'queued.scraper', urls: data['data'].keys
data['agents']['and_chr']['browser'] = 'Android Chrome'
@@ -43,12 +49,9 @@ def build_pages
yield index_page
data['data'].each do |feature_id, feature|
- url = "https://github.com/Fyrd/caniuse/raw/main/features-json/#{feature_id}.json"
-
- response = Request.run(url)
- instrument 'process_response.scraper', response: response
+ path = File.join(CANIUSE_DIRECTORY, 'features-json', "#{feature_id}.json")
- feature = JSON.parse(response.body)
+ feature = JSON.parse(File.read(path))
name = feature['title']
type = feature['categories'].find { |category| name.include?(category) } || feature['categories'].first
diff --git a/lib/docs/scrapers/tensorflow/tensorflow.rb b/lib/docs/scrapers/tensorflow/tensorflow.rb
index 770d35b87c..a54e2ac014 100644
--- a/lib/docs/scrapers/tensorflow/tensorflow.rb
+++ b/lib/docs/scrapers/tensorflow/tensorflow.rb
@@ -45,8 +45,8 @@ class Tensorflow < UrlScraper
end
def get_latest_version(opts)
- doc = fetch_doc(self.base_url, opts)
- doc.title[/TensorFlow v([.\d]+)/, 1]
+ tag = get_github_tags('tensorflow', 'tensorflow', opts).find { |t| !t['name'].include?('-rc') }
+ tag['name'][1..-1]
end
private
diff --git a/lib/docs/scrapers/threejs.rb b/lib/docs/scrapers/threejs.rb
index c3ef9bc25d..7fd3147359 100644
--- a/lib/docs/scrapers/threejs.rb
+++ b/lib/docs/scrapers/threejs.rb
@@ -44,17 +44,17 @@ class Threejs < UrlScraper
self.base_url = "https://threejs.org/docs"
def get_latest_version(opts)
- get_latest_github_release('mrdoob', 'three.js', opts)[1..]
+ get_github_tags('mrdoob', 'three.js', opts).first['name'][1..]
end
def initial_paths
paths = []
- url = 'https://threejs.org/docs/list.json'
+ url = 'https://threejs.org/docs/search.json'
response = Request.run(url)
json_data = JSON.parse(response.body)
# Process both API and manual sections
- process_documentation(json_data['en'], paths)
+ process_documentation(json_data, paths)
paths
end
@@ -78,4 +78,4 @@ def process_documentation(data, paths, prefix = '')
end
end
end
-end
\ No newline at end of file
+end
diff --git a/lib/docs/scrapers/tokio.rb b/lib/docs/scrapers/tokio.rb
new file mode 100644
index 0000000000..6cf56d019e
--- /dev/null
+++ b/lib/docs/scrapers/tokio.rb
@@ -0,0 +1,31 @@
+module Docs
+ class Tokio < UrlScraper
+ self.name = 'Tokio'
+ self.type = 'tokio'
+ self.root_path = 'index.html'
+ self.links = {
+ home: 'https://tokio.rs/',
+ code: 'https://github.com/tokio-rs/tokio'
+ }
+
+ html_filters.push 'tokio/entries', 'tokio/clean_html'
+
+ options[:rate_limit] = 50
+ options[:container] = 'section.content'
+ options[:skip_patterns] = [/\/next|\/\d\.\d*/, /rabbitmq\//]
+ options[:attribution] = <<-HTML
+ MIT License
+ Copyright © Tokio Contributors
+ HTML
+
+ version do
+ self.release = '1.53.1'
+ self.base_url = "https://docs.rs/tokio/#{self.release}/tokio/"
+ end
+
+ def get_latest_version(opts)
+ doc = fetch_doc('https://docs.rs/tokio/latest/tokio/index.html', opts)
+ doc.at_css('h2 > .version').content.strip
+ end
+ end
+end
diff --git a/lib/docs/scrapers/typescript.rb b/lib/docs/scrapers/typescript.rb
index 246d77c9d3..06acbe95c2 100644
--- a/lib/docs/scrapers/typescript.rb
+++ b/lib/docs/scrapers/typescript.rb
@@ -29,12 +29,12 @@ class Typescript < UrlScraper
]
options[:attribution] = <<-HTML
- © 2012-2025 Microsoft
+ © 2012-2026 Microsoft
Licensed under the Apache License, Version 2.0.
HTML
version do
- self.release = '5.9.2'
+ self.release = '6.0.3'
self.base_url = 'https://www.typescriptlang.org/'
end
diff --git a/lib/docs/scrapers/vite.rb b/lib/docs/scrapers/vite.rb
index 1a55691376..70b322f549 100644
--- a/lib/docs/scrapers/vite.rb
+++ b/lib/docs/scrapers/vite.rb
@@ -11,7 +11,7 @@ class Vite < UrlScraper
options[:root_title] = 'Vite'
options[:attribution] = <<-HTML
- © 2019–present, Yuxi (Evan) You and Vite contributors
+ © 2019-present, VoidZero Inc. and Vite contributors
Licensed under the MIT License.
HTML
@@ -22,10 +22,15 @@ class Vite < UrlScraper
html_filters.push 'vite/entries', 'vite/clean_html'
version do
- self.release = '7.1.2'
+ self.release = '8.0.0'
self.base_url = 'https://vite.dev/'
end
+ version '7' do
+ self.release = '7.3.1'
+ self.base_url = 'https://v7.vite.dev/'
+ end
+
version '6' do
self.release = '6.3.5'
self.base_url = 'https://v6.vite.dev/'
diff --git a/lib/docs/scrapers/vitest.rb b/lib/docs/scrapers/vitest.rb
index ab7a3d1bb3..5855de21fb 100644
--- a/lib/docs/scrapers/vitest.rb
+++ b/lib/docs/scrapers/vitest.rb
@@ -13,19 +13,23 @@ class Vitest < UrlScraper
options[:skip] = %w(blog)
options[:attribution] = <<-HTML
- © 2021-Present Anthony Fu
- © 2021-Present Matias Capeletto
+ © 2021-Present VoidZero Inc. and Vitest contributors
Licensed under the MIT License.
HTML
self.initial_paths = %w(guide/)
html_filters.push 'vitest/entries', 'vite/clean_html'
-
+
version do
- self.release = '3.0.8'
+ self.release = '4.1.0'
self.base_url = 'https://vitest.dev/'
end
-
+
+ version '3' do
+ self.release = '3.2.4'
+ self.base_url = 'https://vitest.dev/'
+ end
+
version '2' do
self.release = '2.1.9'
self.base_url = 'https://v2.vitest.dev/'
diff --git a/lib/docs/scrapers/vue.rb b/lib/docs/scrapers/vue.rb
index 158a546b19..79515edcdf 100644
--- a/lib/docs/scrapers/vue.rb
+++ b/lib/docs/scrapers/vue.rb
@@ -14,12 +14,12 @@ class Vue < UrlScraper
options[:replace_paths] = { 'guide/' => 'guide/index.html' }
options[:attribution] = <<-HTML
- © 2013–present Yuxi Evan You
+ © 2018-present, Yuxi (Evan) You and Vue contributors
Licensed under the MIT License.
HTML
version '3' do
- self.release = '3.5.18'
+ self.release = '3.5.38'
self.base_url = 'https://vuejs.org/'
self.initial_paths = %w(guide/introduction.html)
html_filters.push 'vue/entries_v3', 'vue/clean_html'
diff --git a/lib/docs/scrapers/vue_router.rb b/lib/docs/scrapers/vue_router.rb
index 3456fd59db..7e4ee6ed1e 100644
--- a/lib/docs/scrapers/vue_router.rb
+++ b/lib/docs/scrapers/vue_router.rb
@@ -16,10 +16,15 @@ class VueRouter < UrlScraper
]
options[:attribution] = <<-HTML
- © 2013–present Evan You
+ © 2014-present Evan You, Eduardo San Martin Morote
Licensed under the MIT License.
HTML
+ version '5' do
+ self.release = '5.0.2'
+ self.base_url = 'https://router.vuejs.org/'
+ end
+
version '4' do
self.release = '4.0.12'
self.base_url = 'https://next.router.vuejs.org/'
diff --git a/lib/docs/scrapers/vueuse.rb b/lib/docs/scrapers/vueuse.rb
index 31e1a8fb75..7ba0e10c5b 100644
--- a/lib/docs/scrapers/vueuse.rb
+++ b/lib/docs/scrapers/vueuse.rb
@@ -22,7 +22,7 @@ class Vueuse < UrlScraper
Licensed under the MIT License.
HTML
- self.release = '13.6.0'
+ self.release = '14.2.1'
self.base_url = 'https://vueuse.org/'
self.initial_paths = %w(functions.html)
html_filters.push 'vueuse/entries', 'vite/clean_html', 'vueuse/clean_html'
diff --git a/lib/docs/scrapers/yarn.rb b/lib/docs/scrapers/yarn.rb
index 6539c7d5ee..5eec6a7049 100644
--- a/lib/docs/scrapers/yarn.rb
+++ b/lib/docs/scrapers/yarn.rb
@@ -13,7 +13,7 @@ class Yarn < UrlScraper
HTML
version do
- self.release = '4.5.1'
+ self.release = '4.12.0'
self.base_url = 'https://yarnpkg.com/'
self.links = {
home: 'https://yarnpkg.com/',
diff --git a/lib/docs/scrapers/zig.rb b/lib/docs/scrapers/zig.rb
index 2f31658e4e..4a36509ccd 100644
--- a/lib/docs/scrapers/zig.rb
+++ b/lib/docs/scrapers/zig.rb
@@ -2,7 +2,7 @@ module Docs
class Zig < UrlScraper
self.name = 'Zig'
self.type = 'simple'
- self.release = '0.15.2'
+ self.release = '0.16.0'
self.base_url = "https://ziglang.org/documentation/#{self.release}/"
self.links = {
home: 'https://ziglang.org/',
@@ -13,7 +13,7 @@ class Zig < UrlScraper
options[:follow_links] = false
options[:attribution] = <<-HTML
- © 2015–2025, Zig contributors
+ © 2015–2026, Zig contributors
Licensed under the MIT License.
HTML
diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor
index 8dc32d85a9..bf668a3aa0 100644
--- a/lib/tasks/docs.thor
+++ b/lib/tasks/docs.thor
@@ -122,6 +122,7 @@ class DocsCLI < Thor
option :default, type: :boolean
option :installed, type: :boolean
option :all, type: :boolean
+ option :rclone, type: :boolean
def download(*names)
require 'unix_utils'
docs = if options[:default]
@@ -223,7 +224,7 @@ class DocsCLI < Thor
option :amend, type: :boolean
def commit(name)
doc = Docs.find(name, false)
- message = options[:message] || "Update #{doc.name} documentation (#{doc.versions.first.release})"
+ message = options[:message] || "Update #{doc.name} documentation (#{doc.versions.first.release})".delete_suffix(" ()")
amend = " --amend" if options[:amend]
system("git add assets/ *#{name}*") && system("git commit -m '#{message}'#{amend}")
rescue Docs::DocNotFound => error
@@ -235,6 +236,7 @@ class DocsCLI < Thor
puts 'Docs -- BEGIN'
require 'open-uri'
+ require 'net/http'
require 'thread'
docs = Docs.all_versions
@@ -247,18 +249,36 @@ class DocsCLI < Thor
dir = File.join(Docs.store_path, doc.path)
FileUtils.mkpath(dir)
- ['index.json', 'meta.json'].each do |filename|
+ # Only meta.json is needed to build the manifest; clients (and the
+ # service worker precache) load index.json directly from the CDN,
+ # which serves it with CORS headers.
+ ['meta.json'].each do |filename|
json = "https://documents.devdocs.io/#{doc.path}/#{filename}?#{time}"
begin
- URI.open(json, "Accept-Encoding" => "identity") do |file|
- mutex.synchronize do
- path = File.join(dir, filename)
- File.write(path, file.read)
+ attempts = 0
+
+ begin
+ attempts += 1
+
+ URI.open(json, "Accept-Encoding" => "identity") do |file|
+ mutex.synchronize do
+ path = File.join(dir, filename)
+ File.write(path, file.read)
+ end
end
+ rescue Net::OpenTimeout, Net::ReadTimeout => e
+ if attempts <= 3
+ wait_seconds = 2**(attempts - 1)
+ puts "Docs -- Retrying #{json} in #{wait_seconds}s (#{e.class}: #{e.message})"
+ sleep(wait_seconds)
+ retry
+ end
+
+ raise
end
rescue => e
- puts "Docs -- Failed to download #{json}!"
- throw e
+ puts "Docs -- Failed to download #{json} after #{attempts} attempts!"
+ raise
end
end
@@ -347,18 +367,33 @@ class DocsCLI < Thor
def download_doc(doc)
target_path = File.join(Docs.store_path, doc.path)
- URI.open "https://downloads.devdocs.io/#{doc.path}.tar.gz" do |file|
- FileUtils.mkpath(target_path)
- file.close
- tar = UnixUtils.gunzip(file.path)
- dir = UnixUtils.untar(tar)
- FileUtils.rm(tar)
- FileUtils.rm_rf(target_path)
- FileUtils.mv(dir, target_path)
- FileUtils.rm(file.path)
+
+ if options[:rclone]
+ require 'tmpdir'
+ Dir.mktmpdir do |dir|
+ system("rclone copy devdocs:devdocs-downloads/#{doc.path}.tar.gz #{dir}")
+ tar_gz_path = File.join(dir, "#{File.basename(doc.path)}.tar.gz")
+ raise "rclone did not download #{doc.path}.tar.gz (not found on remote?)" unless File.exist?(tar_gz_path)
+ extract_doc(tar_gz_path, target_path)
+ end
+ else
+ URI.open "https://downloads.devdocs.io/#{doc.path}.tar.gz" do |file|
+ file.close
+ extract_doc(file.path, target_path)
+ FileUtils.rm(file.path)
+ end
end
end
+ def extract_doc(tar_gz_path, target_path)
+ FileUtils.mkpath(target_path)
+ tar = UnixUtils.gunzip(tar_gz_path)
+ dir = UnixUtils.untar(tar)
+ FileUtils.rm(tar)
+ FileUtils.rm_rf(target_path)
+ FileUtils.mv(dir, target_path)
+ end
+
def package_doc(doc)
path = File.join Docs.store_path, doc.path
diff --git a/lib/tasks/sprites.thor b/lib/tasks/sprites.thor
index fb1b39aa5d..54df982f07 100644
--- a/lib/tasks/sprites.thor
+++ b/lib/tasks/sprites.thor
@@ -56,6 +56,9 @@ class SpritesCLI < Thor
save_manifest(items, icons_per_row, 'assets/images/sprites/docs.json')
+ # SCSS wrapped in ERB templates must be compiled manually.
+ compile_scss_erb
+
if options[:remove_public_icons]
logger.info('Removing public/icons')
FileUtils.rm_rf('public/icons')
@@ -213,6 +216,18 @@ class SpritesCLI < Thor
"assets/images/sprites/docs#{size == 32 ? '@2x' : ''}.png"
end
+ def compile_scss_erb
+ scss_erb_files = Dir['assets/stylesheets/**/*.scss.erb']
+
+ scss_erb_files.each do |erb_path|
+ scss_path = erb_path.gsub('.erb', '')
+ File.open(scss_path, 'w') do |f|
+ f.write(ERB.new(File.open(erb_path).read).result)
+ logger.info("Compiling #{erb_path} to #{scss_path}")
+ end
+ end
+ end
+
def image_optim
@image_optim ||= ImageOptim.new(
:config_paths => [],
diff --git a/lib/tasks/test.thor b/lib/tasks/test.thor
index e68fec8f26..f001fe7b8c 100644
--- a/lib/tasks/test.thor
+++ b/lib/tasks/test.thor
@@ -26,4 +26,11 @@ class TestCLI < Thor
def app
require 'app_test'
end
+
+ desc 'coverage', 'Coverage report for core tests'
+ def coverage
+ ENV['COVERAGE'] = 'true'
+ require_relative '../../test/test_helper'
+ self.all
+ end
end
diff --git a/public/icons/docs/celery/16.png b/public/icons/docs/celery/16.png
new file mode 100644
index 0000000000..3d582e88bd
Binary files /dev/null and b/public/icons/docs/celery/16.png differ
diff --git a/public/icons/docs/celery/16@2x.png b/public/icons/docs/celery/16@2x.png
new file mode 100644
index 0000000000..fda9845140
Binary files /dev/null and b/public/icons/docs/celery/16@2x.png differ
diff --git a/public/icons/docs/celery/SOURCE b/public/icons/docs/celery/SOURCE
new file mode 100644
index 0000000000..2f071f4820
--- /dev/null
+++ b/public/icons/docs/celery/SOURCE
@@ -0,0 +1 @@
+https://docs.celeryq.dev/en/stable/_static/favicon.ico
diff --git a/public/icons/docs/coldfusion/16.png b/public/icons/docs/coldfusion/16.png
new file mode 100644
index 0000000000..00ccc8f85e
Binary files /dev/null and b/public/icons/docs/coldfusion/16.png differ
diff --git a/public/icons/docs/coldfusion/16@2x.png b/public/icons/docs/coldfusion/16@2x.png
new file mode 100644
index 0000000000..888db9e321
Binary files /dev/null and b/public/icons/docs/coldfusion/16@2x.png differ
diff --git a/public/icons/docs/coldfusion/SOURCE b/public/icons/docs/coldfusion/SOURCE
new file mode 100644
index 0000000000..32de636dcb
--- /dev/null
+++ b/public/icons/docs/coldfusion/SOURCE
@@ -0,0 +1 @@
+https://cfdocs.org/apple-touch-icon.png
diff --git a/public/icons/docs/couchdb/16.png b/public/icons/docs/couchdb/16.png
new file mode 100644
index 0000000000..7cae0c8b51
Binary files /dev/null and b/public/icons/docs/couchdb/16.png differ
diff --git a/public/icons/docs/couchdb/16@2x.png b/public/icons/docs/couchdb/16@2x.png
new file mode 100644
index 0000000000..81dc55b202
Binary files /dev/null and b/public/icons/docs/couchdb/16@2x.png differ
diff --git a/public/icons/docs/couchdb/SOURCE b/public/icons/docs/couchdb/SOURCE
new file mode 100644
index 0000000000..86560405ed
--- /dev/null
+++ b/public/icons/docs/couchdb/SOURCE
@@ -0,0 +1 @@
+https://docs.couchdb.org/en/stable/_static/favicon.ico
diff --git a/public/icons/docs/cyclejs/16.png b/public/icons/docs/cyclejs/16.png
new file mode 100644
index 0000000000..b870a7a678
Binary files /dev/null and b/public/icons/docs/cyclejs/16.png differ
diff --git a/public/icons/docs/cyclejs/16@2x.png b/public/icons/docs/cyclejs/16@2x.png
new file mode 100644
index 0000000000..7e8be668a4
Binary files /dev/null and b/public/icons/docs/cyclejs/16@2x.png differ
diff --git a/public/icons/docs/cyclejs/SOURCE b/public/icons/docs/cyclejs/SOURCE
new file mode 100644
index 0000000000..eb89497c99
--- /dev/null
+++ b/public/icons/docs/cyclejs/SOURCE
@@ -0,0 +1 @@
+https://cycle.js.org/img/cyclejs_logo.svg
diff --git a/public/icons/docs/maplibre_gl/16.png b/public/icons/docs/maplibre_gl/16.png
new file mode 100644
index 0000000000..00c1db8283
Binary files /dev/null and b/public/icons/docs/maplibre_gl/16.png differ
diff --git a/public/icons/docs/maplibre_gl/16@2x.png b/public/icons/docs/maplibre_gl/16@2x.png
new file mode 100644
index 0000000000..49f4b9e988
Binary files /dev/null and b/public/icons/docs/maplibre_gl/16@2x.png differ
diff --git a/public/icons/docs/maplibre_gl/SOURCE b/public/icons/docs/maplibre_gl/SOURCE
new file mode 100644
index 0000000000..ddd48e6326
--- /dev/null
+++ b/public/icons/docs/maplibre_gl/SOURCE
@@ -0,0 +1 @@
+SOURCE: https://raw.githubusercontent.com/maplibre/maplibre-gl-js/main/docs/assets/logo.svg
diff --git a/public/icons/docs/minitest/16.png b/public/icons/docs/minitest/16.png
new file mode 100644
index 0000000000..dd04a16cc9
Binary files /dev/null and b/public/icons/docs/minitest/16.png differ
diff --git a/public/icons/docs/minitest/16@2x.png b/public/icons/docs/minitest/16@2x.png
new file mode 100644
index 0000000000..2992e604c1
Binary files /dev/null and b/public/icons/docs/minitest/16@2x.png differ
diff --git a/public/icons/docs/minitest/SOURCE b/public/icons/docs/minitest/SOURCE
new file mode 100644
index 0000000000..6c36bfe062
--- /dev/null
+++ b/public/icons/docs/minitest/SOURCE
@@ -0,0 +1,2 @@
+https://minite.st/favicon-16x16.png
+https://minite.st/favicon-32x32.png
diff --git a/public/icons/docs/odin/16.png b/public/icons/docs/odin/16.png
new file mode 100644
index 0000000000..038d7d801b
Binary files /dev/null and b/public/icons/docs/odin/16.png differ
diff --git a/public/icons/docs/odin/16@2x.png b/public/icons/docs/odin/16@2x.png
new file mode 100644
index 0000000000..d4a3e83320
Binary files /dev/null and b/public/icons/docs/odin/16@2x.png differ
diff --git a/public/icons/docs/odin/SOURCE b/public/icons/docs/odin/SOURCE
new file mode 100644
index 0000000000..167c103d26
--- /dev/null
+++ b/public/icons/docs/odin/SOURCE
@@ -0,0 +1 @@
+https://github.com/odin-lang/odin-lang.org/blob/master/static/favicon.svg
\ No newline at end of file
diff --git a/public/icons/docs/odin_packages/16.png b/public/icons/docs/odin_packages/16.png
new file mode 100644
index 0000000000..038d7d801b
Binary files /dev/null and b/public/icons/docs/odin_packages/16.png differ
diff --git a/public/icons/docs/odin_packages/16@2x.png b/public/icons/docs/odin_packages/16@2x.png
new file mode 100644
index 0000000000..d4a3e83320
Binary files /dev/null and b/public/icons/docs/odin_packages/16@2x.png differ
diff --git a/public/icons/docs/odin_packages/SOURCE b/public/icons/docs/odin_packages/SOURCE
new file mode 100644
index 0000000000..167c103d26
--- /dev/null
+++ b/public/icons/docs/odin_packages/SOURCE
@@ -0,0 +1 @@
+https://github.com/odin-lang/odin-lang.org/blob/master/static/favicon.svg
\ No newline at end of file
diff --git a/public/icons/docs/opentofu/16.png b/public/icons/docs/opentofu/16.png
new file mode 100644
index 0000000000..2c18bbedef
Binary files /dev/null and b/public/icons/docs/opentofu/16.png differ
diff --git a/public/icons/docs/opentofu/16@2x.png b/public/icons/docs/opentofu/16@2x.png
new file mode 100644
index 0000000000..073cc5053a
Binary files /dev/null and b/public/icons/docs/opentofu/16@2x.png differ
diff --git a/public/icons/docs/opentofu/SOURCE b/public/icons/docs/opentofu/SOURCE
new file mode 100644
index 0000000000..893b013fe2
--- /dev/null
+++ b/public/icons/docs/opentofu/SOURCE
@@ -0,0 +1,2 @@
+https://github.com/opentofu/opentofu.org/blob/main/static/favicons/favicon-16x16.png
+https://github.com/opentofu/opentofu.org/blob/main/static/favicons/favicon-32x32.png
diff --git a/public/icons/docs/polars/16.png b/public/icons/docs/polars/16.png
new file mode 100644
index 0000000000..c005950aa8
Binary files /dev/null and b/public/icons/docs/polars/16.png differ
diff --git a/public/icons/docs/polars/16@2x.png b/public/icons/docs/polars/16@2x.png
new file mode 100644
index 0000000000..940e5427f8
Binary files /dev/null and b/public/icons/docs/polars/16@2x.png differ
diff --git a/public/icons/docs/polars/SOURCE b/public/icons/docs/polars/SOURCE
new file mode 100644
index 0000000000..c63c050877
--- /dev/null
+++ b/public/icons/docs/polars/SOURCE
@@ -0,0 +1 @@
+https://raw.githubusercontent.com/pola-rs/polars-static/master/icons/favicon-32x32.png
diff --git a/public/icons/docs/powershell/16.png b/public/icons/docs/powershell/16.png
new file mode 100644
index 0000000000..730b58a302
Binary files /dev/null and b/public/icons/docs/powershell/16.png differ
diff --git a/public/icons/docs/powershell/16@2x.png b/public/icons/docs/powershell/16@2x.png
new file mode 100644
index 0000000000..5840c2dafd
Binary files /dev/null and b/public/icons/docs/powershell/16@2x.png differ
diff --git a/public/icons/docs/powershell/SOURCE b/public/icons/docs/powershell/SOURCE
new file mode 100644
index 0000000000..3604afa785
--- /dev/null
+++ b/public/icons/docs/powershell/SOURCE
@@ -0,0 +1 @@
+https://learn.microsoft.com/en-us/powershell/media/index/ps_black_128.svg
diff --git a/public/icons/docs/rabbitmq/16.png b/public/icons/docs/rabbitmq/16.png
new file mode 100644
index 0000000000..0f99414aab
Binary files /dev/null and b/public/icons/docs/rabbitmq/16.png differ
diff --git a/public/icons/docs/rabbitmq/16@2x.png b/public/icons/docs/rabbitmq/16@2x.png
new file mode 100644
index 0000000000..e0b7027dfa
Binary files /dev/null and b/public/icons/docs/rabbitmq/16@2x.png differ
diff --git a/public/icons/docs/rabbitmq/SOURCE b/public/icons/docs/rabbitmq/SOURCE
new file mode 100644
index 0000000000..98720bdbd4
--- /dev/null
+++ b/public/icons/docs/rabbitmq/SOURCE
@@ -0,0 +1 @@
+https://www.rabbitmq.com/assets/files/rabbitmq-logo-e91cacd38fcef5219149bc5cfa10b384.svg
diff --git a/public/icons/docs/tokio/16.png b/public/icons/docs/tokio/16.png
new file mode 100644
index 0000000000..830b4021d5
Binary files /dev/null and b/public/icons/docs/tokio/16.png differ
diff --git a/public/icons/docs/tokio/16@2x.png b/public/icons/docs/tokio/16@2x.png
new file mode 100644
index 0000000000..f593b8573d
Binary files /dev/null and b/public/icons/docs/tokio/16@2x.png differ
diff --git a/public/icons/docs/tokio/SOURCE b/public/icons/docs/tokio/SOURCE
new file mode 100644
index 0000000000..e09093716f
--- /dev/null
+++ b/public/icons/docs/tokio/SOURCE
@@ -0,0 +1 @@
+https://upload.wikimedia.org/wikipedia/commons/6/60/Tokio_logo.svg
diff --git a/public/icons/docs/vite/16.png b/public/icons/docs/vite/16.png
index 90ffa78658..a081c7b8c1 100644
Binary files a/public/icons/docs/vite/16.png and b/public/icons/docs/vite/16.png differ
diff --git a/public/icons/docs/vite/16@2x.png b/public/icons/docs/vite/16@2x.png
index 375fb680ed..e8d199489b 100644
Binary files a/public/icons/docs/vite/16@2x.png and b/public/icons/docs/vite/16@2x.png differ
diff --git a/public/icons/docs/vite/SOURCE b/public/icons/docs/vite/SOURCE
index f1fe46eefe..f7b304f1f2 100644
--- a/public/icons/docs/vite/SOURCE
+++ b/public/icons/docs/vite/SOURCE
@@ -1 +1 @@
-https://vitejs.dev/logo.svg
+https://vite.dev/logo-without-border.svg
diff --git a/public/icons/docs/vitest/16.png b/public/icons/docs/vitest/16.png
index fd44a6bb81..7fbe9bee38 100644
Binary files a/public/icons/docs/vitest/16.png and b/public/icons/docs/vitest/16.png differ
diff --git a/public/icons/docs/vitest/16@2x.png b/public/icons/docs/vitest/16@2x.png
index 051339ca94..6b72ed130d 100644
Binary files a/public/icons/docs/vitest/16@2x.png and b/public/icons/docs/vitest/16@2x.png differ
diff --git a/public/icons/docs/vitest/SOURCE b/public/icons/docs/vitest/SOURCE
index 911a2163a4..0cf0aaabab 100644
--- a/public/icons/docs/vitest/SOURCE
+++ b/public/icons/docs/vitest/SOURCE
@@ -1 +1,2 @@
https://vitest.dev/logo.svg
+https://vitest.dev/favicon.ico
diff --git a/test/app_test.rb b/test/app_test.rb
index 44fca64aab..4b5e4c3297 100644
--- a/test/app_test.rb
+++ b/test/app_test.rb
@@ -46,21 +46,11 @@ def app
end
describe "/[static-page]" do
- it "redirects to /#/[static-page] by default" do
- %w(offline about news help).each do |page|
+ it "renders the app shell directly" do
+ %w(settings offline about news help).each do |page|
get "/#{page}", {}, 'HTTP_USER_AGENT' => MODERN_BROWSER
- assert last_response.redirect?
- assert_equal "https://example.org/#/#{page}", last_response['Location']
- end
- end
-
- it "redirects via JS cookie when a cookie exists" do
- %w(offline about news help).each do |page|
- set_cookie('foo=bar')
- get "/#{page}", {}, 'HTTP_USER_AGENT' => MODERN_BROWSER
- assert last_response.redirect?
- assert_equal 'https://example.org/', last_response['Location']
- assert last_response['Set-Cookie'].start_with?("initial_path=%2F#{page}; path=/; expires=")
+ assert last_response.ok?
+ assert_nil last_response['Set-Cookie']
end
end
end
@@ -90,12 +80,11 @@ def app
assert last_response.ok?
end
- it "redirects via JS cookie when the doc exists and is enabled" do
+ it "renders when the doc exists and is enabled" do
set_cookie('docs=html~5')
get '/html~5/', {}, 'HTTP_USER_AGENT' => MODERN_BROWSER
- assert last_response.redirect?
- assert_equal 'https://example.org/', last_response['Location']
- assert last_response['Set-Cookie'].start_with?("initial_path=%2Fhtml%7E5%2F; path=/; expires=")
+ assert last_response.ok?
+ assert_nil last_response['Set-Cookie']
end
it "renders when the doc exists, has no version in the path, and isn't enabled" do
@@ -103,12 +92,11 @@ def app
assert last_response.ok?
end
- it "redirects via JS cookie when the doc exists, has no version in the path, and a version is enabled" do
+ it "renders when the doc exists, has no version in the path, and a version is enabled" do
set_cookie('docs=html~5')
get '/html/', {}, 'HTTP_USER_AGENT' => MODERN_BROWSER
- assert last_response.redirect?
- assert_equal 'https://example.org/', last_response['Location']
- assert last_response['Set-Cookie'].start_with?("initial_path=%2Fhtml%2F; path=/; expires=")
+ assert last_response.ok?
+ assert_nil last_response['Set-Cookie']
end
it "renders when the doc exists and is enabled, and the request is from Googlebot" do
diff --git a/test/assets/search_hash_test.js b/test/assets/search_hash_test.js
new file mode 100644
index 0000000000..276f3df387
--- /dev/null
+++ b/test/assets/search_hash_test.js
@@ -0,0 +1,50 @@
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const test = require("node:test");
+const vm = require("node:vm");
+
+const context = {
+ app: {
+ config: { search_param: "q" },
+ router: { replaceHash: () => {} },
+ views: {},
+ View: class {},
+ },
+ location: { hash: "" },
+ $: {
+ urlDecodeFragment: decodeURIComponent,
+ },
+};
+
+vm.createContext(context);
+
+for (const file of [
+ "assets/javascripts/views/search/search_scope.js",
+ "assets/javascripts/views/search/search.js",
+]) {
+ vm.runInContext(fs.readFileSync(file, "utf8"), context, {
+ filename: file,
+ });
+}
+
+test("URL search hash preserves plus signs in a scoped C++ query", () => {
+ context.location.hash = "#q=c++%20std::min";
+
+ const scope = new context.app.views.SearchScope();
+ let replacedHash;
+ context.app.router.replaceHash = (hash) => {
+ replacedHash = hash;
+ };
+
+ assert.equal(scope.getHashValue(), "c++");
+ assert.equal(scope.extractHashValue(), "c++");
+ assert.equal(replacedHash, "#q=std::min");
+});
+
+test("URL search hash preserves encoded literal plus signs in the query", () => {
+ context.location.hash = "#q=operator%2B";
+
+ const search = new context.app.views.Search();
+
+ assert.equal(search.getHashValue(), "operator+");
+});
diff --git a/test/lib/docs/core/doc_test.rb b/test/lib/docs/core/doc_test.rb
index 6b41cdd9c8..b6c08c5f4f 100644
--- a/test/lib/docs/core/doc_test.rb
+++ b/test/lib/docs/core/doc_test.rb
@@ -1,5 +1,6 @@
require_relative '../../../test_helper'
require_relative '../../../../lib/docs'
+require 'base64'
class DocsDocTest < Minitest::Spec
let :doc do
@@ -237,6 +238,18 @@ class DocsDocTest < Minitest::Spec
doc.store_page(store, 'id')
end
end
+
+ context "when the store can't be opened" do
+ before do
+ stub(store).open { raise Docs::SetupError, "oh no" }
+ end
+
+ it "logs an error" do
+ output = capture_io { doc.store_page(store, 'id') }.first
+
+ assert_match "ERROR: oh no", output
+ end
+ end
end
describe ".store_pages" do
@@ -353,6 +366,18 @@ class DocsDocTest < Minitest::Spec
doc.store_pages(store)
end
end
+
+ context "when the store can't be opened" do
+ before do
+ stub(store).replace { raise Docs::SetupError, "oh no" }
+ end
+
+ it "logs an error" do
+ output = capture_io { doc.store_pages(store) }.first
+
+ assert_match "ERROR: oh no", output
+ end
+ end
end
describe ".versions" do
@@ -406,4 +431,372 @@ class DocsDocTest < Minitest::Spec
assert_equal "Outdated minor version", instance.outdated_state('2006-01-02', '2006-02-03')
end
end
+
+ # PRIVATE CLASS METHODS
+
+ describe ".default_slug" do
+ it "returns name.downcase when name is alphanumeric" do
+ doc.name = 'FooBar'
+ assert_equal 'foobar', doc.send(:default_slug)
+ end
+
+ it "returns name.downcase when name contains underscores" do
+ doc.name = 'Foo_Bar'
+ assert_equal 'foo_bar', doc.send(:default_slug)
+ end
+
+ it "returns nil when name contains a hyphen" do
+ doc.name = 'Foo-Bar'
+ assert_nil doc.send(:default_slug)
+ end
+
+ it "returns nil when name contains a space" do
+ doc.name = 'Foo Bar'
+ assert_nil doc.send(:default_slug)
+ end
+
+ it "returns nil when name contains a dot" do
+ doc.name = 'Foo.Bar'
+ assert_nil doc.send(:default_slug)
+ end
+ end
+
+ describe ".store_page?" do
+ it "returns true when page has non-empty :entries" do
+ assert doc.send(:store_page?, page)
+ end
+
+ it "returns false when page has empty :entries" do
+ refute doc.send(:store_page?, page.merge(entries: []))
+ end
+
+ it "returns false when page has nil :entries" do
+ refute doc.send(:store_page?, page.merge(entries: nil))
+ end
+ end
+
+ describe ".store_index" do
+ let :index do
+ obj = Object.new
+ stub(obj).to_json { '{"entries":[]}' }
+ obj
+ end
+
+ context "when read_write is true (default)" do
+ it "reads old JSON from store" do
+ mock(store).read('index.json') { '{}' }
+ stub(store).write
+ doc.send(:store_index, store, 'index.json', index)
+ end
+
+ it "writes new JSON to store" do
+ stub(store).read { '{}' }
+ mock(store).write('index.json', '{"entries":[]}')
+ doc.send(:store_index, store, 'index.json', index)
+ end
+ end
+
+ context "when read_write is false" do
+ it "doesn't read from store" do
+ dont_allow(store).read
+ doc.send(:store_index, store, 'index.json', index, false)
+ end
+
+ it "doesn't write to store" do
+ dont_allow(store).write
+ doc.send(:store_index, store, 'index.json', index, false)
+ end
+ end
+ end
+
+ describe ".store_meta" do
+ it "writes to META_FILENAME" do
+ stub(store).size { 0 }
+ mock(store).write(Docs::Doc::META_FILENAME, anything)
+ doc.send(:store_meta, store)
+ end
+
+ it "includes :mtime as an integer in the written JSON" do
+ stub(store).size { 0 }
+ stub(store).write(Docs::Doc::META_FILENAME, anything) do |_, json|
+ @json = JSON.parse(json)
+ end
+ doc.send(:store_meta, store)
+ assert @json.key?('mtime')
+ assert_instance_of Integer, @json['mtime']
+ end
+
+ it "includes :db_size from the store" do
+ stub(store).size(Docs::Doc::DB_FILENAME) { 42 }
+ stub(store).write(Docs::Doc::META_FILENAME, anything) do |_, json|
+ @json = JSON.parse(json)
+ end
+ doc.send(:store_meta, store)
+ assert_equal 42, @json['db_size']
+ end
+ end
+
+ # PRIVATE INSTANCE METHODS
+
+ describe "#fetch" do
+ let :instance do
+ doc.new
+ end
+
+ let :logger do
+ l = Object.new
+ stub(l).debug
+ stub(l).error
+ l
+ end
+
+ let :opts do
+ { logger: logger }
+ end
+
+ let :response do
+ r = Object.new
+ stub(r).success? { true }
+ stub(r).body { 'body' }
+ r
+ end
+
+ before do
+ stub(Docs::Request).run { response }
+ end
+
+ it "returns the response body on success" do
+ assert_equal 'body', instance.send(:fetch, 'http://example.com/', opts)
+ end
+
+ it "calls Request.run with the given url" do
+ mock(Docs::Request).run('http://example.com/', anything) { response }
+ instance.send(:fetch, 'http://example.com/', opts)
+ end
+
+ it "adds Authorization header when opts has :github_token and url is GitHub API" do
+ opts[:github_token] = 'mytoken'
+ mock(Docs::Request).run(anything, anything) do |_, request_opts|
+ assert_equal 'token mytoken', request_opts[:headers]['Authorization']
+ response
+ end
+ instance.send(:fetch, 'https://api.github.com/repos', opts)
+ end
+
+ it "doesn't add Authorization header for non-GitHub API urls even with :github_token" do
+ opts[:github_token] = 'mytoken'
+ mock(Docs::Request).run(anything, anything) do |_, request_opts|
+ refute request_opts[:headers].key?('Authorization')
+ response
+ end
+ instance.send(:fetch, 'http://example.com/', opts)
+ end
+
+ context "when response is unsuccessful" do
+ before do
+ stub(response).success? { false }
+ stub(response).timed_out? { false }
+ stub(response).code { 404 }
+ end
+
+ it "raises an error" do
+ assert_raises(RuntimeError) do
+ instance.send(:fetch, 'http://example.com/', opts)
+ end
+ end
+
+ it "includes the response code in the error message" do
+ err = assert_raises(RuntimeError) { instance.send(:fetch, 'http://example.com/', opts) }
+ assert_match '404', err.message
+ end
+ end
+
+ context "when response times out" do
+ before do
+ stub(response).success? { false }
+ stub(response).timed_out? { true }
+ end
+
+ it "raises an error with a timeout message" do
+ err = assert_raises(RuntimeError) { instance.send(:fetch, 'http://example.com/', opts) }
+ assert_match 'Timed out', err.message
+ end
+ end
+ end
+
+ describe "#fetch_doc" do
+ let :instance do
+ doc.new
+ end
+
+ let :opts do
+ { logger: nil }
+ end
+
+ it "returns a Nokogiri::HTML::Document" do
+ stub(instance).fetch { '' }
+ assert_instance_of Nokogiri::HTML::Document, instance.send(:fetch_doc, 'http://example.com/', opts)
+ end
+
+ it "fetches the given url" do
+ mock(instance).fetch('http://example.com/', opts) { '' }
+ instance.send(:fetch_doc, 'http://example.com/', opts)
+ end
+ end
+
+ describe "#fetch_json" do
+ let :instance do
+ doc.new
+ end
+
+ let :opts do
+ { logger: nil }
+ end
+
+ it "returns parsed JSON" do
+ stub(instance).fetch { '{"key":"value"}' }
+ assert_equal({ 'key' => 'value' }, instance.send(:fetch_json, 'http://example.com/', opts))
+ end
+
+ it "fetches the given url" do
+ mock(instance).fetch('http://example.com/', opts) { '{}' }
+ instance.send(:fetch_json, 'http://example.com/', opts)
+ end
+ end
+
+ describe "#get_npm_version" do
+ let :instance do
+ doc.new
+ end
+
+ let :opts do
+ { logger: nil }
+ end
+
+ it "returns the 'latest' dist-tag by default" do
+ stub(instance).fetch_json { { 'dist-tags' => { 'latest' => '1.2.3' } } }
+ assert_equal '1.2.3', instance.send(:get_npm_version, 'some-package', opts)
+ end
+
+ it "returns a specific dist-tag when given" do
+ stub(instance).fetch_json { { 'dist-tags' => { 'next' => '2.0.0-beta' } } }
+ assert_equal '2.0.0-beta', instance.send(:get_npm_version, 'some-package', opts, 'next')
+ end
+
+ it "fetches from the npm registry URL" do
+ mock(instance).fetch_json('https://registry.npmjs.com/some-package', opts) { { 'dist-tags' => { 'latest' => '1.0.0' } } }
+ instance.send(:get_npm_version, 'some-package', opts)
+ end
+ end
+
+ describe "#get_latest_github_release" do
+ let :instance do
+ doc.new
+ end
+
+ let :opts do
+ { logger: nil }
+ end
+
+ it "strips a leading 'v' from the tag name" do
+ stub(instance).fetch_json { { 'tag_name' => 'v1.2.3' } }
+ assert_equal '1.2.3', instance.send(:get_latest_github_release, 'owner', 'repo', opts)
+ end
+
+ it "returns the tag name unchanged when it doesn't start with 'v'" do
+ stub(instance).fetch_json { { 'tag_name' => '1.2.3' } }
+ assert_equal '1.2.3', instance.send(:get_latest_github_release, 'owner', 'repo', opts)
+ end
+
+ it "fetches from the correct GitHub releases URL" do
+ mock(instance).fetch_json('https://api.github.com/repos/owner/repo/releases/latest', opts) { { 'tag_name' => '1.0.0' } }
+ instance.send(:get_latest_github_release, 'owner', 'repo', opts)
+ end
+ end
+
+ describe "#get_github_tags" do
+ let :instance do
+ doc.new
+ end
+
+ let :opts do
+ { logger: nil }
+ end
+
+ it "fetches from the correct GitHub tags URL" do
+ mock(instance).fetch_json('https://api.github.com/repos/owner/repo/tags', opts) { [] }
+ instance.send(:get_github_tags, 'owner', 'repo', opts)
+ end
+
+ it "returns the tags list" do
+ tags = [{ 'name' => 'v1.0' }]
+ stub(instance).fetch_json { tags }
+ assert_equal tags, instance.send(:get_github_tags, 'owner', 'repo', opts)
+ end
+ end
+
+ describe "#get_github_file_contents" do
+ let :instance do
+ doc.new
+ end
+
+ let :opts do
+ { logger: nil }
+ end
+
+ it "returns Base64-decoded file contents" do
+ stub(instance).fetch_json { { 'content' => ::Base64.encode64('hello world') } }
+ assert_equal 'hello world', instance.send(:get_github_file_contents, 'owner', 'repo', 'README.md', opts)
+ end
+
+ it "fetches from the correct GitHub contents URL" do
+ mock(instance).fetch_json('https://api.github.com/repos/owner/repo/contents/README.md', opts) { { 'content' => ::Base64.encode64('') } }
+ instance.send(:get_github_file_contents, 'owner', 'repo', 'README.md', opts)
+ end
+ end
+
+ describe "#get_latest_github_commit_date" do
+ let :instance do
+ doc.new
+ end
+
+ let :opts do
+ { logger: nil }
+ end
+
+ it "returns the epoch timestamp of the latest commit" do
+ date_str = '2023-06-15T12:00:00Z'
+ commits = [{ 'commit' => { 'author' => { 'date' => date_str } } }]
+ stub(instance).fetch_json { commits }
+ assert_equal Date.iso8601(date_str).to_time.to_i, instance.send(:get_latest_github_commit_date, 'owner', 'repo', opts)
+ end
+
+ it "fetches from the correct GitHub commits URL" do
+ date_str = '2023-06-15T12:00:00Z'
+ commits = [{ 'commit' => { 'author' => { 'date' => date_str } } }]
+ mock(instance).fetch_json('https://api.github.com/repos/owner/repo/commits', opts) { commits }
+ instance.send(:get_latest_github_commit_date, 'owner', 'repo', opts)
+ end
+ end
+
+ describe "#get_gitlab_tags" do
+ let :instance do
+ doc.new
+ end
+
+ let :opts do
+ { logger: nil }
+ end
+
+ it "fetches from the correct GitLab tags URL" do
+ mock(instance).fetch_json('https://gitlab.example.com/api/v4/projects/group%2Fproject/repository/tags', opts) { [] }
+ instance.send(:get_gitlab_tags, 'gitlab.example.com', 'group', 'project', opts)
+ end
+
+ it "returns the tags list" do
+ tags = [{ 'name' => 'v1.0' }]
+ stub(instance).fetch_json { tags }
+ assert_equal tags, instance.send(:get_gitlab_tags, 'gitlab.example.com', 'group', 'project', opts)
+ end
+ end
end
diff --git a/test/lib/docs/core/entry_index_test.rb b/test/lib/docs/core/entry_index_test.rb
index 95495eb90a..6222d4e163 100644
--- a/test/lib/docs/core/entry_index_test.rb
+++ b/test/lib/docs/core/entry_index_test.rb
@@ -154,6 +154,72 @@ class DocsEntryIndexTest < Minitest::Spec
end
end
+ describe "#sort_fn" do
+ def sort(a, b)
+ index.send(:sort_fn, a, b)
+ end
+
+ context "when neither string starts with a digit 1-9" do
+ it "uses case-insensitive string comparison" do
+ assert_operator sort('apple', 'banana'), :<, 0
+ assert_operator sort('banana', 'apple'), :>, 0
+ end
+
+ it "returns 0 for case-insensitive equal strings" do
+ assert_equal 0, sort('Apple', 'apple')
+ end
+
+ it "treats strings starting with '0' as non-numeric" do
+ assert_equal '0apple'.casecmp('banana'), sort('0apple', 'banana')
+ end
+ end
+
+ context "when at least one string starts with 1-9 but neither contains version dots" do
+ it "uses case-insensitive string comparison" do
+ # lexicographic: '1' < '2', so '10 Test' < '2 Test'
+ assert_operator sort('10 Test', '2 Test'), :<, 0
+ end
+
+ it "returns 0 for case-insensitive equal strings" do
+ assert_equal 0, sort('2 Test', '2 test')
+ end
+ end
+
+ context "when one string has version dots and the other does not" do
+ it "sorts the dotted string before the undotted one" do
+ assert_operator sort('1.2. Test', '2 Test'), :<, 0
+ end
+
+ it "sorts the undotted string after the dotted one" do
+ assert_operator sort('2 Test', '1.2. Test'), :>, 0
+ end
+
+ it "sorts the dotted string before a non-numeric string" do
+ assert_operator sort('1.2. Test', 'abc'), :<, 0
+ end
+ end
+
+ context "when both strings have version dots" do
+ it "sorts numerically rather than lexicographically" do
+ assert_operator sort('1.9. Test', '1.10. Test'), :<, 0
+ assert_operator sort('4.20. Test', '4.3. Test'), :>, 0
+ end
+
+ it "pads shorter version depths with zeros before comparing" do
+ assert_operator sort('4. Test', '4.2. Test'), :<, 0
+ assert_operator sort('4.2. Test', '4. Test'), :>, 0
+ end
+
+ it "compares multi-level versions against shorter ones correctly" do
+ assert_operator sort('4.2.2. Test', '4.3. Test'), :<, 0
+ end
+
+ it "returns 0 for identical version strings" do
+ assert_equal 0, sort('1.2. Test', '1.2. Test')
+ end
+ end
+ end
+
describe "#to_json" do
it "returns the JSON string for #as_json" do
stub(index).as_json { { entries: [1], types: [2] } }
diff --git a/test/lib/docs/core/filter_test.rb b/test/lib/docs/core/filter_test.rb
index 10872c7beb..205c6bced1 100644
--- a/test/lib/docs/core/filter_test.rb
+++ b/test/lib/docs/core/filter_test.rb
@@ -8,6 +8,12 @@ class DocsFilterTest < Minitest::Spec
before do
context[:base_url] = 'http://example.com/path'
context[:url] = 'http://example.com/path/file'
+ context[:version] = '6.1'
+ context[:release] = '6.1.1'
+ context[:links] = {
+ home: 'homunculus',
+ code: 'codified'
+ }
end
describe "#subpath" do
@@ -55,6 +61,24 @@ def slug(subpath)
end
end
+ describe "#version" do
+ it "returns the version" do
+ assert_equal filter.version, "6.1"
+ end
+ end
+
+ describe "#release" do
+ it "returns the release" do
+ assert_equal filter.release, "6.1.1"
+ end
+ end
+
+ describe "#links" do
+ it "returns the links hash" do
+ assert_equal filter.links, { home: 'homunculus', code: 'codified' }
+ end
+ end
+
describe "#root_page?" do
it "returns true when #subpath is blank" do
stub(filter).subpath { '' }
@@ -176,4 +200,18 @@ def slug(subpath)
refute filter.absolute_url_string?('http')
end
end
+
+ describe "#clean_path" do
+ it "replaces some special characters with dashes" do
+ bad_path = "/horrible!url:design;huh".freeze
+ good_path = "/horrible-url-design-huh".freeze
+ assert_equal filter.clean_path(bad_path), good_path
+ end
+
+ it "replaces + with the word 'plus'" do
+ bad_path = "/c++".freeze
+ good_path = "/c_plus__plus_".freeze
+ assert_equal filter.clean_path(bad_path), good_path
+ end
+ end
end
diff --git a/test/lib/docs/core/scraper_test.rb b/test/lib/docs/core/scraper_test.rb
index 7b5522b3cb..e2efebf1af 100644
--- a/test/lib/docs/core/scraper_test.rb
+++ b/test/lib/docs/core/scraper_test.rb
@@ -510,4 +510,166 @@ class Scraper < Docs::Scraper
assert_equal Docs, pipeline.instrumentation_service
end
end
+
+ # PRIVATE INSTANCE METHODS
+
+ describe "#url_for" do
+ it "returns the root url when path is empty" do
+ assert_equal scraper.root_url.to_s, scraper.send(:url_for, '')
+ end
+
+ it "returns the root url when path is '/'" do
+ assert_equal scraper.root_url.to_s, scraper.send(:url_for, '/')
+ end
+
+ it "joins base_url and path for a non-root path" do
+ stub(Scraper).base_url { 'http://example.com/' }
+ assert_equal 'http://example.com/page', scraper.send(:url_for, '/page')
+ end
+ end
+
+ describe "#parse" do
+ it "returns a two-element array" do
+ response.body = 'just a fragment'
+ _, title = scraper.send(:parse, response)
+ assert_nil title
+ end
+ end
+
+ describe "#pipeline_context" do
+ it "returns options merged with the response url" do
+ stub(scraper).options { { base_url: 'http://example.com/' } }
+ response.url = 'http://example.com/page'
+ context = scraper.send(:pipeline_context, response)
+ assert_equal 'http://example.com/page', context[:url]
+ assert_equal 'http://example.com/', context[:base_url]
+ end
+
+ it "does not mutate the options hash" do
+ opts = { base_url: 'http://example.com/' }
+ stub(scraper).options { opts }
+ response.url = 'url'
+ scraper.send(:pipeline_context, response)
+ refute opts.key?(:url)
+ end
+ end
+
+ describe "#process_response" do
+ before do
+ stub(scraper).parse(response) { ['', 'My Title'] }
+ response.url = 'http://example.com/'
+ end
+
+ it "returns a hash" do
+ assert_instance_of Hash, scraper.send(:process_response, response)
+ end
+
+ it "passes :html_title from parse into the pipeline context" do
+ mock(scraper.pipeline).call(anything, anything, anything) do |_, context, _|
+ assert_equal 'My Title', context[:html_title]
+ end
+ scraper.send(:process_response, response)
+ end
+ end
+
+ describe "#with_filters" do
+ it "uses only the given filters inside the block" do
+ scraper.send(:with_filters) do
+ @filters = scraper.pipeline.filters
+ end
+ assert_equal [], @filters
+ end
+
+ it "resets the pipeline after the block" do
+ pipeline_before = scraper.pipeline
+ scraper.send(:with_filters) {}
+ refute_same pipeline_before, scraper.pipeline
+ end
+ end
+
+ describe "#additional_options" do
+ it "returns an empty hash" do
+ assert_equal({}, scraper.send(:additional_options))
+ end
+ end
+
+ class FixedScraper < Scraper
+ include Docs::Scraper::FixInternalUrlsBehavior
+ end
+
+ describe "FixInternalUrlsBehavior" do
+ let :fixed_scraper do
+ FixedScraper.new.tap { |s| s.extend FakeInstrumentation }
+ end
+
+ describe ".with_internal_urls (private)" do
+ it "sets .internal_urls to the result of fetch_internal_urls during the block" do
+ any_instance_of(FixedScraper) do |instance|
+ stub(instance).fetch_internal_urls { ['url1', 'url2'] }
+ end
+ FixedScraper.send(:with_internal_urls) do
+ @urls = FixedScraper.internal_urls
+ end
+ assert_equal ['url1', 'url2'], @urls
+ end
+
+ it "clears .internal_urls after the block" do
+ any_instance_of(FixedScraper) do |instance|
+ stub(instance).fetch_internal_urls { [] }
+ end
+ FixedScraper.send(:with_internal_urls) {}
+ assert_nil FixedScraper.internal_urls
+ end
+ end
+
+ describe "#additional_options (private)" do
+ it "returns an empty hash when .internal_urls is nil" do
+ stub(FixedScraper).internal_urls { nil }
+ assert_equal({}, fixed_scraper.send(:additional_options))
+ end
+
+ it "returns fixed_internal_urls: true when .internal_urls is set" do
+ stub(FixedScraper).internal_urls { ['url'] }
+ assert fixed_scraper.send(:additional_options)[:fixed_internal_urls]
+ end
+
+ it "sets :only to a Set of .internal_urls" do
+ stub(FixedScraper).internal_urls { ['url1', 'url2'] }
+ assert_equal Set.new(['url1', 'url2']), fixed_scraper.send(:additional_options)[:only]
+ end
+
+ it "sets :only_patterns to nil" do
+ stub(FixedScraper).internal_urls { ['url'] }
+ assert_nil fixed_scraper.send(:additional_options)[:only_patterns]
+ end
+
+ it "sets :skip to nil" do
+ stub(FixedScraper).internal_urls { ['url'] }
+ assert_nil fixed_scraper.send(:additional_options)[:skip]
+ end
+ end
+
+ describe "#process_response (private)" do
+ before do
+ stub(fixed_scraper).parse(response) { ['', nil] }
+ response.url = 'http://example.com/page'
+ end
+
+ it "merges :response_url from the response into the result" do
+ result = fixed_scraper.send(:process_response, response)
+ assert_equal 'http://example.com/page', result[:response_url]
+ end
+ end
+ end
end
diff --git a/test/lib/docs/core/scrapers/url_scraper_test.rb b/test/lib/docs/core/scrapers/url_scraper_test.rb
index 77fc235d57..7469ce4070 100644
--- a/test/lib/docs/core/scrapers/url_scraper_test.rb
+++ b/test/lib/docs/core/scrapers/url_scraper_test.rb
@@ -85,6 +85,38 @@ class Scraper < Docs::UrlScraper
stub(Docs::Requester).run { 'response' }
assert_equal 'response', result
end
+
+ context "when options[:rate_limit] is set" do
+ before do
+ @saved_limiter = Docs::UrlScraper.class_variable_get(:@@rate_limiter)
+ Docs::UrlScraper.class_variable_set(:@@rate_limiter, nil)
+ stub(Scraper).options { { rate_limit: 5 } }
+ stub(Docs::Requester).run
+ end
+
+ after do
+ Docs::UrlScraper.class_variable_set(:@@rate_limiter, @saved_limiter)
+ end
+
+ it "creates a RateLimiter with the given limit" do
+ stub(Typhoeus).before
+ scraper.send :request_all, 'urls'
+ assert_equal 5, Docs::UrlScraper.class_variable_get(:@@rate_limiter).limit
+ end
+
+ it "registers the RateLimiter with Typhoeus.before" do
+ mock(Typhoeus).before
+ scraper.send :request_all, 'urls'
+ end
+
+ it "updates the existing RateLimiter's limit instead of creating a new one" do
+ existing = Docs::UrlScraper::RateLimiter.new(3)
+ Docs::UrlScraper.class_variable_set(:@@rate_limiter, existing)
+ dont_allow(Typhoeus).before
+ scraper.send :request_all, 'urls'
+ assert_equal 5, existing.limit
+ end
+ end
end
describe "#process_response?" do
@@ -120,4 +152,34 @@ class Scraper < Docs::UrlScraper
assert result
end
end
+
+ describe "RateLimiter" do
+ let :limiter do
+ Docs::UrlScraper::RateLimiter.new(3)
+ end
+
+ describe "#initialize" do
+ it "sets the limit" do
+ assert_equal 3, limiter.limit
+ end
+ end
+
+ describe "#limit=" do
+ it "updates the limit" do
+ limiter.limit = 10
+ assert_equal 10, limiter.limit
+ end
+ end
+
+ describe "#to_proc" do
+ it "returns a Proc" do
+ assert_instance_of Proc, limiter.to_proc
+ end
+
+ it "returns a proc that calls #call" do
+ mock(limiter).call
+ limiter.to_proc.call
+ end
+ end
+ end
end
diff --git a/test/lib/docs/core/subscriber_test.rb b/test/lib/docs/core/subscriber_test.rb
new file mode 100644
index 0000000000..8f56cb9853
--- /dev/null
+++ b/test/lib/docs/core/subscriber_test.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative '../../../test_helper'
+require_relative '../../../../lib/docs'
+
+class DocsSubscriberTest < Minitest::Spec
+ let(:subscriber) { Docs::Subscriber.new }
+
+ describe "#format_url" do
+ it "removes http:// scheme" do
+ assert_equal 'devdocs.test.co/path', subscriber.send(:format_url, 'http://devdocs.test.co/path')
+ end
+
+ it "removes https:// scheme" do
+ assert_equal 'devdocs.test.co/path', subscriber.send(:format_url, 'https://devdocs.test.co/path')
+ end
+
+ it "returns non-http urls unchanged" do
+ assert_equal 'devdocs.test.co/path', subscriber.send(:format_url, 'devdocs.test.co/path')
+ end
+ end
+
+ describe "#format_path" do
+ it "removes the current working directory prefix" do
+ full_path = File.join(Dir.pwd, 'some/file.rb')
+ assert_equal 'some/file.rb', subscriber.send(:format_path, full_path)
+ end
+
+ it "returns paths outside the cwd unchanged" do
+ assert_equal '/other/path', subscriber.send(:format_path, '/other/path')
+ end
+
+ it "calls to_s on the argument" do
+ path = Object.new
+ def path.to_s; '/other/path'; end
+ assert_equal '/other/path', subscriber.send(:format_path, path)
+ end
+ end
+
+ describe "#justify" do
+ describe "when terminal_width is nil" do
+ before { subscriber.instance_variable_set(:@terminal_width, nil) }
+
+ it "returns the string unchanged" do
+ assert_equal 'hello', subscriber.send(:justify, 'hello')
+ end
+ end
+
+ describe "when terminal_width is set" do
+ before { subscriber.instance_variable_set(:@terminal_width, 20) }
+
+ it "pads a short string to terminal width" do
+ result = subscriber.send(:justify, 'hi')
+ assert_equal 20, result.length
+ assert result.start_with?('hi')
+ end
+
+ it "truncates a long string to terminal width" do
+ result = subscriber.send(:justify, 'a' * 30)
+ assert_equal 20, result.length
+ end
+
+ it "reserves space for a trailing tag and pads the rest" do
+ result = subscriber.send(:justify, 'hi [tag]')
+ assert_equal 20, result.length
+ assert result.end_with?('[tag]')
+ end
+
+ it "truncates the non-tag part when the string is too long" do
+ result = subscriber.send(:justify, ('a' * 30) + ' [tag]')
+ assert_equal 20, result.length
+ assert result.end_with?('[tag]')
+ end
+ end
+ end
+
+ describe "#terminal_width" do
+ it "returns nil when stdout is not a tty" do
+ old_stdout = $stdout
+ $stdout = StringIO.new
+ sub = Docs::Subscriber.new
+ assert_nil sub.send(:terminal_width)
+ ensure
+ $stdout = old_stdout
+ end
+
+ it "reads COLUMNS env var when stdout is a tty" do
+ sub = Docs::Subscriber.new
+ # Override tty? on the instance to avoid needing a real tty
+ def sub.tty?; true; end
+ old_columns = ENV.delete('COLUMNS')
+ ENV['COLUMNS'] = '666'
+ assert_equal 666, sub.send(:terminal_width)
+ ensure
+ ENV['COLUMNS'] = old_columns
+ end
+ end
+end
diff --git a/test/lib/docs/filters/core/attribution_test.rb b/test/lib/docs/filters/core/attribution_test.rb
new file mode 100644
index 0000000000..73ed53e912
--- /dev/null
+++ b/test/lib/docs/filters/core/attribution_test.rb
@@ -0,0 +1,39 @@
+require_relative '../../../../test_helper'
+require_relative '../../../../../lib/docs'
+
+class AttributionFilterTest < Minitest::Spec
+ include FilterTestHelper
+ self.filter_class = Docs::AttributionFilter
+ self.filter_type = 'html'
+
+ before do
+ @body = "Hello!"
+ end
+
+ context "when attribution is a string" do
+ before do
+ context[:attribution] = "Copyright 2033 Mark Twain"
+ context[:base_url] = "q"
+ end
+
+ it "adds it to the document" do
+ assert_equal filter_output.at_css('#mtwain').inner_html, "Copyright 2033 Mark Twain"
+ refute_nil filter_output.at_css('p._attribution-p'), "Attribution node should exist"
+ refute_nil filter_output.at_css('.doc'), "Existing doc should still be present"
+ refute_nil filter_output.at_css('a._attribution-link'), "Attribution link should exist"
+ end
+ end
+
+ context "when attribution is a Proc" do
+ before do
+ context[:attribution] = ->(filter) {
+ "Copyright 2034 Mark Twain"
+ }
+ context[:base_url] = "q"
+ end
+
+ it "adds its output to the document" do
+ assert_equal filter_output.at_css('#mtwain').inner_html, "Copyright 2034 Mark Twain"
+ end
+ end
+end
diff --git a/test/lib/docs/filters/core/clean_local_urls_test.rb b/test/lib/docs/filters/core/clean_local_urls_test.rb
new file mode 100644
index 0000000000..e2a2e8eb67
--- /dev/null
+++ b/test/lib/docs/filters/core/clean_local_urls_test.rb
@@ -0,0 +1,22 @@
+require_relative '../../../../test_helper'
+require_relative '../../../../../lib/docs'
+
+class CleanLocalUrlsFilterTest < Minitest::Spec
+ include FilterTestHelper
+ self.filter_class = Docs::CleanLocalUrlsFilter
+
+ it "removes images/iframes referencing localhost" do
+ context[:base_url] = 'http://localhost'
+ @body = '
'
+ assert_empty filter_output.css('img, iframe')
+ end
+
+ it "turns links referencing localhost into plain text" do
+ context[:base_url] = 'http://localhost'
+ @body = 'Link'
+ node = filter_output.at_css('span')
+ assert node, 'expected a node'
+ refute node['href']
+ assert_equal 'Link', node.text
+ end
+end
diff --git a/test/lib/docs/filters/core/images_test.rb b/test/lib/docs/filters/core/images_test.rb
new file mode 100644
index 0000000000..2763b18cfe
--- /dev/null
+++ b/test/lib/docs/filters/core/images_test.rb
@@ -0,0 +1,134 @@
+require_relative '../../../../test_helper'
+require_relative '../../../../../lib/docs'
+require 'ostruct'
+
+class ImagesFilterTest < Minitest::Spec
+ include FilterTestHelper
+ self.filter_class = Docs::ImagesFilter
+
+ IMG_SRC = 'http://example.com/img.png'
+ IMG_BODY = '
'
+
+ before do
+ Docs::ImagesFilter.cache.clear
+ context[:optimize_images] = false
+ end
+
+ def make_response(success: true, mime_type: 'image/png', content_length: 100, body: 'imgdata', code: 200)
+ OpenStruct.new(success?: success, mime_type: mime_type, content_length: content_length, body: body, code: code)
+ end
+
+ def stub_request(response)
+ stub(Docs::Request).run { |_url, &block| block.call(response) }
+ end
+
+ context "when download_images is false" do
+ it "returns doc unchanged and makes no requests" do
+ context[:download_images] = false
+ @body = IMG_BODY
+ dont_allow(Docs::Request).run
+ assert_equal IMG_SRC, filter_output.at_css('img')['src']
+ end
+ end
+
+ context "with a data: URL" do
+ it "skips the image and makes no request" do
+ @body = '
'
+ dont_allow(Docs::Request).run
+ assert_equal 'data:image/png;base64,aaaaaa2320345bbb234', filter_output.at_css('img')['src']
+ end
+ end
+
+ context "with a non-http(s) URL" do
+ it "skips the image and makes no request" do
+ @body = '
'
+ dont_allow(Docs::Request).run
+ filter_output
+ end
+ end
+
+ context "with a successful image response" do
+ it "replaces src with a base64 data URI" do
+ @body = IMG_BODY
+ stub_request make_response(body: 'imgdata', mime_type: 'image/png', content_length: 7)
+ expected = "data:image/png;base64,#{Base64.strict_encode64('imgdata')}"
+ assert_equal expected, filter_output.at_css('img')['src']
+ end
+
+ it "caches the data URI for subsequent calls" do
+ @body = IMG_BODY
+ stub_request make_response(body: 'imgdata', mime_type: 'image/png', content_length: 7)
+ filter_output
+ assert Docs::ImagesFilter.cache.key?(IMG_SRC)
+ assert Docs::ImagesFilter.cache[IMG_SRC].start_with?('data:')
+ end
+ end
+
+ context "when response is not successful" do
+ it "instruments broken.image" do
+ @body = IMG_BODY
+ stub_request make_response(success: false, code: 404)
+ @called = false
+ filter.subscribe('broken.image') { @called = true }
+ filter_output
+ assert @called
+ end
+
+ it "does not alter the src" do
+ @body = IMG_BODY
+ stub_request make_response(success: false, code: 404)
+ assert_equal IMG_SRC, filter_output.at_css('img')['src']
+ end
+ end
+
+ context "when response mime type is not image/" do
+ it "instruments invalid.image" do
+ @body = IMG_BODY
+ stub_request make_response(mime_type: 'text/html')
+ @called = false
+ filter.subscribe('invalid.image') { @called = true }
+ filter_output
+ assert @called
+ end
+ end
+
+ context "when an exception is raised during the request" do
+ it "instruments error.image" do
+ @body = IMG_BODY
+ stub(Docs::Request).run { raise 'connection error' }
+ @called = false
+ filter.subscribe('error.image') { @called = true }
+ filter_output
+ assert @called
+ end
+ end
+
+ context "with a cached src" do
+ it "uses the cached data URI without making a request" do
+ cached = "data:image/png;base64,#{Base64.strict_encode64('cached')}"
+ Docs::ImagesFilter.cache[IMG_SRC] = cached
+ @body = IMG_BODY
+ dont_allow(Docs::Request).run
+ assert_equal cached, filter_output.at_css('img')['src']
+ end
+
+ it "leaves src unchanged when cache marks the URL as failed" do
+ Docs::ImagesFilter.cache[IMG_SRC] = false
+ @body = IMG_BODY
+ dont_allow(Docs::Request).run
+ assert_equal IMG_SRC, filter_output.at_css('img')['src']
+ end
+ end
+
+ context "when optimize_images is not disabled" do
+ it "passes image data through optimize_image_data" do
+ @body = IMG_BODY
+ optimized = 'optimized_imgdata'
+ stub_request make_response(body: 'imgdata', mime_type: 'image/png', content_length: 7)
+ stub(Docs::ImagesFilter).optimize_image_data('imgdata') { optimized }
+ context.delete(:optimize_images)
+ expected = "data:image/png;base64,#{Base64.strict_encode64(optimized)}"
+ assert_equal expected, filter_output.at_css('img')['src']
+ end
+ end
+end
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 8ae02364c6..602d158bdd 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -1,3 +1,13 @@
+# Flag set by Thor coverage task.
+if ENV['COVERAGE'] == 'true'
+ require 'simplecov'
+
+ # Focus only on core app files.
+ SimpleCov.add_filter {|src| !(src.filename =~ /\/core\//)}
+
+ SimpleCov.start 'rails'
+end
+
ENV['RACK_ENV'] = 'test'
require 'bundler/setup'
diff --git a/views/unsupported.erb b/views/unsupported.erb
index 6c1ded7ef1..3540adddb7 100644
--- a/views/unsupported.erb
+++ b/views/unsupported.erb
@@ -20,7 +20,7 @@
We decided to prioritize speed and new features over support for older browsers.
- — @DevDocs + — DevDocs
