diff --git a/.gitignore b/.gitignore index 8b22282640..01a28e203f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .bundle *.pxm *.sketch +*.pyc tmp public/assets public/fonts diff --git a/Gemfile b/Gemfile index 2dafff05d3..ba366107ae 100644 --- a/Gemfile +++ b/Gemfile @@ -1,12 +1,15 @@ source 'https://rubygems.org' -ruby '2.2.0' +ruby '2.3.0' +gem 'rake' gem 'thor' gem 'pry', '~> 0.10.0' -gem 'activesupport', '~> 4.0', require: false +gem 'activesupport', '~> 4.2', require: false gem 'yajl-ruby', require: false group :app do + gem 'capybara' + gem 'selenium-webdriver' gem 'rack' gem 'sinatra' gem 'sinatra-contrib' @@ -29,19 +32,15 @@ end group :docs do gem 'typhoeus' - gem 'nokogiri', '~> 1.6.0' + gem 'nokogiri' gem 'html-pipeline' gem 'progress_bar', require: false gem 'unix_utils', require: false + gem 'tty-pager', require: false end group :test do gem 'minitest' gem 'rr', require: false gem 'rack-test', require: false -end - -if ENV['SELENIUM'] == '1' - gem 'capybara' - gem 'selenium-webdriver' -end +end \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index bf835bd818..e6ed08136f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,110 +1,118 @@ GEM remote: https://rubygems.org/ specs: - activesupport (4.2.0) + activesupport (4.2.6) i18n (~> 0.7) json (~> 1.7, >= 1.7.7) minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - backports (3.6.4) + backports (3.6.8) better_errors (2.1.1) coderay (>= 1.0.0) erubis (>= 2.6.6) rack (>= 0.9.0) - browser (0.8.0) - coderay (1.1.0) - coffee-script (2.3.0) + browser (2.0.3) + coderay (1.1.1) + coffee-script (2.4.1) coffee-script-source execjs - coffee-script-source (1.8.0) - daemons (1.1.9) + coffee-script-source (1.10.0) + concurrent-ruby (1.0.1) + daemons (1.2.3) erubis (2.7.0) - ethon (0.7.2) + ethon (0.8.1) ffi (>= 1.3.0) - eventmachine (1.0.4) - execjs (2.2.2) - ffi (1.9.6) - highline (1.6.21) - hike (1.2.3) - html-pipeline (1.11.0) - activesupport (>= 2) - nokogiri (~> 1.4) + eventmachine (1.2.0.1) + execjs (2.6.0) + ffi (1.9.10) + highline (1.7.8) + html-pipeline (2.4.0) + activesupport (>= 2, < 5) + nokogiri (>= 1.4) i18n (0.7.0) - json (1.8.2) + json (1.8.3) method_source (0.8.2) - mini_portile (0.6.2) - minitest (5.5.1) - multi_json (1.10.1) - nokogiri (1.6.5) - mini_portile (~> 0.6.0) + mini_portile2 (2.0.0) + minitest (5.8.4) + multi_json (1.11.2) + nokogiri (1.6.7.2) + mini_portile2 (~> 2.0.0.rc2) options (2.3.2) - progress_bar (1.0.3) - highline (~> 1.6.1) + progress_bar (1.0.5) + highline (~> 1.6) options (~> 2.3.0) - pry (0.10.1) + pry (0.10.3) coderay (~> 1.1.0) method_source (~> 0.8.1) slop (~> 3.4) - rack (1.6.0) + rack (1.6.4) rack-protection (1.5.3) rack rack-test (0.6.3) rack (>= 1.0) + rake (11.1.2) rr (1.1.2) - sass (3.4.9) - sinatra (1.4.5) - rack (~> 1.4) + sass (3.4.22) + sinatra (1.4.7) + rack (~> 1.5) rack-protection (~> 1.4) - tilt (~> 1.3, >= 1.3.4) - sinatra-contrib (1.4.2) + tilt (>= 1.3, < 3) + sinatra-contrib (1.4.7) backports (>= 2.0) multi_json rack-protection rack-test sinatra (~> 1.4.0) - tilt (~> 1.3) + tilt (>= 1.3, < 3) slop (3.6.0) - sprockets (2.12.3) - hike (~> 1.2) - multi_json (~> 1.0) - rack (~> 1.0) - tilt (~> 1.1, != 1.3.0) - sprockets-helpers (1.1.0) - sprockets (~> 2.0) - thin (1.6.3) + sprockets (3.6.0) + concurrent-ruby (~> 1.0) + rack (> 1, < 3) + sprockets-helpers (1.2.1) + sprockets (>= 2.2) + thin (1.6.4) daemons (~> 1.0, >= 1.0.9) - eventmachine (~> 1.0) + eventmachine (~> 1.0, >= 1.0.4) rack (~> 1.0) thor (0.19.1) - thread_safe (0.3.4) - tilt (1.4.1) - typhoeus (0.7.0) - ethon (>= 0.7.1) + thread_safe (0.3.5) + tilt (2.0.2) + tty-pager (0.4.0) + tty-screen (~> 0.5.0) + tty-which (~> 0.1.0) + verse (~> 0.4.0) + tty-screen (0.5.0) + tty-which (0.1.0) + typhoeus (1.0.1) + ethon (>= 0.8.0) tzinfo (1.2.2) thread_safe (~> 0.1) - uglifier (2.7.0) - execjs (>= 0.3.0) - json (>= 1.8.0) + uglifier (3.0.0) + execjs (>= 0.3.0, < 3) + unicode_utils (1.4.0) unix_utils (0.0.15) + verse (0.4.0) + unicode_utils (~> 1.4.0) yajl-ruby (1.2.1) PLATFORMS ruby DEPENDENCIES - activesupport (~> 4.0) + activesupport (~> 4.2) better_errors browser coffee-script erubis html-pipeline minitest - nokogiri (~> 1.6.0) + nokogiri progress_bar pry (~> 0.10.0) rack rack-test + rake rr sass sinatra @@ -113,7 +121,11 @@ DEPENDENCIES sprockets-helpers thin thor + tty-pager typhoeus uglifier unix_utils yajl-ruby + +BUNDLED WITH + 1.11.2 \ No newline at end of file diff --git a/assets/stylesheets/application.css.scss b/assets/stylesheets/application.css.scss index 9e246856f8..05328ddcf9 100644 --- a/assets/stylesheets/application.css.scss +++ b/assets/stylesheets/application.css.scss @@ -28,33 +28,31 @@ 'components/prism', 'components/mobile'; -@import 'pages/angular', +@import 'pages/base', + 'pages/angular', + 'pages/apache', 'pages/bower', 'pages/c', 'pages/chai', + 'pages/clojure', 'pages/coffeescript', - 'pages/cordova', 'pages/d3', 'pages/ember', 'pages/express', + 'pages/git', 'pages/go', 'pages/haskell', 'pages/jquery', 'pages/knockout', - 'pages/git', - 'pages/grunt', 'pages/laravel', - 'pages/less', - 'pages/lodash', - 'pages/marionette', - 'pages/markdown', - 'pages/maxcdn', + 'pages/lua', 'pages/mdn', + 'pages/meteor', 'pages/modernizr', 'pages/moment', - 'pages/mongoose', 'pages/nginx', 'pages/node', + 'pages/npm', 'pages/php', 'pages/phpunit', 'pages/postgres', @@ -64,7 +62,6 @@ 'pages/requirejs', 'pages/rethinkdb', 'pages/rfc', - 'pages/sinon', 'pages/socketio', 'pages/sphinx', 'pages/underscore', diff --git a/assets/stylesheets/components/_content.scss b/assets/stylesheets/components/_content.scss index 16c4f3f186..fc3e54d104 100644 --- a/assets/stylesheets/components/_content.scss +++ b/assets/stylesheets/components/_content.scss @@ -205,15 +205,16 @@ float: right; max-width: 15em; margin: .25rem 0 1.5rem 1.5rem; - padding: .75rem 1rem; + padding: .625rem 1rem; @extend %box; - + ._lined-heading { margin-top: 0; } + + h1, + ._lined-heading { margin-top: 0; } } ._toc-title { - margin: 0 0 .75em; + margin: 0 0 .5rem; font-size: inherit; + font-weight: bold; } ._toc-list { @@ -222,6 +223,8 @@ list-style: none; } +._toc-link { @extend %internal-link; } + // // Static page // @@ -425,6 +428,11 @@ ._splash-sponsor > ._maxcdn-logo { background-image: image-url('maxcdn-bw.png'); + @if $style == 'dark' { + filter: invert(100%); + -webkit-filter: invert(100%); + } + @media (-webkit-min-device-pixel-ratio: 1.5), (min-resolution: 144dpi) { background-image: image-url('maxcdn-bw@2x.png'); } @@ -443,6 +451,11 @@ ._splash-sponsor > ._shopify-logo { background-image: image-url('shopify-bw.png'); + @if $style == 'dark' { + filter: invert(100%); + -webkit-filter: invert(100%); + } + @media (-webkit-min-device-pixel-ratio: 1.5), (min-resolution: 144dpi) { background-image: image-url('shopify-bw@2x.png'); } diff --git a/assets/stylesheets/components/_header.scss b/assets/stylesheets/components/_header.scss index 39db0b4fe9..4c96aae661 100644 --- a/assets/stylesheets/components/_header.scss +++ b/assets/stylesheets/components/_header.scss @@ -98,7 +98,11 @@ top: 1rem; left: 1rem; opacity: .4; - @extend %icon, %icon-search; + @if $style == 'dark' { + @extend %icon, %icon-search-white; + } @else { + @extend %icon, %icon-search; + } } } @@ -132,7 +136,11 @@ opacity: .3; &:hover { opacity: .5; } - &:before { @extend %icon, %icon-clear; } + @if $style == 'dark' { + &:before { @extend %icon, %icon-close-white; } + } @else { + &:before { @extend %icon, %icon-clear; } + } ._search-active > & { display: block; } } diff --git a/assets/stylesheets/components/_mobile.scss b/assets/stylesheets/components/_mobile.scss index 96b41158e9..25af1a2bcd 100644 --- a/assets/stylesheets/components/_mobile.scss +++ b/assets/stylesheets/components/_mobile.scss @@ -36,8 +36,8 @@ float: none; width: auto; overflow: hidden; - padding-left: 0; - padding-right: 0; + padding-left: 2px; + padding-right: 2px; border-right: 0; &:before { left: .5rem; } diff --git a/assets/stylesheets/components/_page.scss b/assets/stylesheets/components/_page.scss index 30935d85a3..a7670f77a1 100644 --- a/assets/stylesheets/components/_page.scss +++ b/assets/stylesheets/components/_page.scss @@ -3,6 +3,10 @@ // ._page { + position: relative; + + ._page-error { position: static; } + > h1 { @extend ._lined-heading; } > h1:first-child { margin-top: 0; } @@ -22,6 +26,35 @@ } } +// +// Links +// + +._links { + position: absolute; + top: 0; + right: 0; + margin: 0; + line-height: 2em; + text-align: right; + + + h1 { margin-top: 0; } + + @media (max-width: 1023px) { display: none; } +} + +._links-link { + display: inline-block; + vertical-align: top; + padding: 0 .5rem; + background: $contentBackground; + @extend %internal-link; + + & + & { margin-left: .75rem; } + &:first-child { padding-left: 1rem; } + &:last-child { padding-right: 0; } +} + // // Attribution box // @@ -29,7 +62,7 @@ ._attribution { clear: both; margin: 2rem 0 1.5rem; - font-size: .75rem; + font-size: 80%; color: $textColorLight; text-align: center; -webkit-font-smoothing: subpixel-antialiased; diff --git a/assets/stylesheets/components/_prism.scss b/assets/stylesheets/components/_prism.scss index 43d6020daf..36926c56f4 100644 --- a/assets/stylesheets/components/_prism.scss +++ b/assets/stylesheets/components/_prism.scss @@ -17,7 +17,11 @@ .token.constant, .token.symbol, .token.deleted { - color: #905; + @if $style == 'dark' { + color: #eb8160; + } @else { + color: #905; + } } .token.selector, @@ -26,7 +30,11 @@ .token.char, .token.builtin, .token.inserted { - color: #5e8e01; + @if $style == 'dark' { + color: #ddcf88; + } @else { + color: #5e8e01; + } } .token.operator, @@ -34,23 +42,39 @@ .token.url, .language-css .token.string, .style .token.string { - color: #a67f59; + @if $style == 'dark' { + color: #b1c676; + } @else { + color: #a67f59; + } } .token.atrule, .token.attr-value, .token.keyword { - color: #0070a3; + @if $style == 'dark' { + color: #91b3ed; + } @else { + color: #0070a3; + } } .token.function { - color: #dd4a68; + @if $style == 'dark' { + color: #c79e6b; + } @else { + color: #dd4a68; + } } .token.regex, .token.important, .token.variable { - color: #e90; + @if $style == 'dark' { + color: #e9c062; + } @else { + color: #e90; + } } .token.important { diff --git a/assets/stylesheets/components/_sidebar.scss b/assets/stylesheets/components/_sidebar.scss index 0b5c7aa488..d029d27496 100644 --- a/assets/stylesheets/components/_sidebar.scss +++ b/assets/stylesheets/components/_sidebar.scss @@ -35,6 +35,17 @@ a:focus { outline: 0; } } +._resizer { + position: absolute; + z-index: $sidebarZ + 1; + top: $headerHeight; + bottom: 0; + left: $sidebarWidth; + margin-left: -2px; + width: 3px; + cursor: col-resize; +} + // // List // @@ -109,11 +120,9 @@ } } -._list-count { +._list-count, ._list-enable { float: right; font-size: .75rem; - color: $textColorLighter; - pointer-events: none; .focus > &, .active > & { @@ -121,6 +130,22 @@ } } +._list-count { + color: $textColorLighter; + pointer-events: none; + + ._list-disabled:hover > & { display: none; } +} + +._list-enable { + display: none; + color: $linkColor; + cursor: pointer; + + &:hover { text-decoration: underline; } + ._list-disabled:hover > & { display: block; } +} + // // List hierarchy // @@ -148,7 +173,11 @@ &:hover { opacity: .65; } &:before { - @extend %icon, %icon-dir; + @if $style == 'dark' { + @extend %icon, %icon-dir-white; + } @else { + @extend %icon, %icon-dir; + } .open > &, .open-title > & { -webkit-transform: rotate(90deg); @@ -214,19 +243,20 @@ } } -._list-noresults { +// +// List note +// + +._list-note { padding: .5rem .75rem; line-height: 1.25rem; font-size: .8125rem; color: $textColorLight; -} -._list-noresults-note { - display: block; - margin-top: .25rem; + & + & { padding-top: 0; } } -._list-noresults-link { cursor: pointer; } +._list-note-link { cursor: pointer; } // // List hover clone @@ -325,6 +355,7 @@ ._sidebar-footer-link { position: relative; display: block; + overflow: hidden; height: 2.5rem; line-height: 1rem; padding: .75rem; @@ -345,7 +376,31 @@ } ._sidebar-footer-edit { - &:before { @extend %icon-settings; } + @if $style == 'dark' { + &:before { @extend %icon-settings-white; } + } @else { + &:before { @extend %icon-settings; } + } +} + +._sidebar-footer-light { + float: right; + width: 2.5rem; + padding: 0; + opacity: .6; + + &:before { + float: none; + position: absolute; + top: .75rem; + left: .75rem; + + @if $style == 'dark' { + @extend %icon-light-white; + } @else { + @extend %icon-light; + } + } } ._sidebar-footer-save { @@ -355,5 +410,9 @@ box-shadow: inset 0 1px $noteGreenBorder, 1px 0 $noteGreenBorder; - &:before { @extend %icon-check; } + @if $style == 'dark' { + &:before { @extend %icon-check-white; } + } @else { + &:before { @extend %icon-check; } + } } diff --git a/assets/stylesheets/global/_base.scss b/assets/stylesheets/global/_base.scss index b69f16949a..728e234077 100644 --- a/assets/stylesheets/global/_base.scss +++ b/assets/stylesheets/global/_base.scss @@ -1,24 +1,3 @@ -html { - height: 100%; - font-size: 100%; - background: $contentBackground; - - @media #{$mediumScreen} { font-size: 93.75%; } -} - -body { - height: 100%; - margin: 0; - overflow: auto; - font: normal 1em/1.7 $baseFont; - color: $textColor; - word-wrap: break-word; - overflow-wrap: break-word; - -webkit-tap-highlight-color: rgba(black, 0); - -webkit-touch-callout: none; - -webkit-text-size-adjust: 100%; - -ms-text-size-adjust: 100%; -} a { color: $linkColor; @@ -55,7 +34,7 @@ b, strong { font-weight: bold; } small { font-size: .9em; } -ul, ol { +._page ul, ._page ol { margin: 1.5em 0; padding: 0 0 0 2em; list-style: disc outside; @@ -66,8 +45,8 @@ ol { list-style-type: decimal; } ol ol { list-style-type: lower-alpha; } ol ol ol { list-style-type: lower-roman; } -li + li { margin-top: .25em; } li > ul, li > ol, dd > ul, dd > ol { margin: .5em 0; } +li > p { margin-bottom: .25em; } dl { margin: 1.5em 0; } dt { font-weight: bold; } @@ -89,6 +68,7 @@ pre, code, samp, %pre, %code { font-family: $monoFont; font-weight: normal; font-style: normal; + font-size: .9em; color: $textColor; white-space: pre-wrap; -moz-tab-size: 2; @@ -101,7 +81,6 @@ pre, %pre { padding: .375rem .75rem; line-height: 1.5; overflow: auto; - font-size: .9em; @extend %box; } @@ -148,6 +127,10 @@ td { + td { border-left: 1px solid $boxBorderLight; } tr:last-child > & { border-bottom: 0; } + > pre:only-child, > p:only-child, > ul:only-child, > ol:only-child { + margin-top: 0; + margin-bottom: 0; + } > pre:last-child, > p:last-child, > ul:last-child, > ol:last-child { margin-bottom: 0; } } diff --git a/assets/stylesheets/global/_classes.scss b/assets/stylesheets/global/_classes.scss index 7e6bff788c..bdc301e369 100644 --- a/assets/stylesheets/global/_classes.scss +++ b/assets/stylesheets/global/_classes.scss @@ -54,7 +54,6 @@ %label { margin: 0 1px; padding: 1px 4px 2px; - background: $labelBackground; border-radius: 3px; } @@ -95,14 +94,9 @@ // %external-link { - &:after { - display: inline-block; - width: .5rem; - height: .4375rem; - margin: .125rem 0 0 .0625rem; - vertical-align: top; - @extend %icon, %icon-link; - } + background: url(../img/icon_external_link.png)center right no-repeat; + padding-right: 13px; + padding-top: 10px; } %internal-link:after { content: none !important; } diff --git a/assets/stylesheets/global/_icons.scss b/assets/stylesheets/global/_icons.scss index d3a2852165..0c968c4606 100644 --- a/assets/stylesheets/global/_icons.scss +++ b/assets/stylesheets/global/_icons.scss @@ -4,76 +4,100 @@ width: 1rem; height: 1rem; background-image: image-url('icons.png'); - background-size: 5rem 13rem; + background-size: 10rem 8rem; } @media (-webkit-min-device-pixel-ratio: 1.5), (min-resolution: 144dpi) { %icon { background-image: image-url('icons@2x.png'); } } +%darkIconFix { + @if $style == 'dark' { + filter: invert(100%) grayscale(100%); + -webkit-filter: invert(100%) grayscale(100%); + } +} + %icon-dir { background-position: 0 0; } %icon-search { background-position: -1rem 0; } %icon-link { background-position: -2.25rem -.25rem; } %icon-clear { background-position: -3rem 0; } %icon-close-white { background-position: -4rem 0; } -%icon-settings { background-position: 0 -1rem; } -%icon-check { background-position: -1rem -1rem; } -._icon-http:before { background-position: -2rem -1rem; } -._icon-jquery:before { background-position: -3rem -1rem; } -._icon-underscore:before { background-position: -4rem -1rem; } -._icon-html:before { background-position: 0 -2rem; } -._icon-css:before { background-position: -1rem -2rem; } -._icon-dom:before { background-position: -2rem -2rem; } -._icon-dom_events:before { background-position: -3rem -2rem; } -._icon-javascript:before { background-position: -4rem -2rem; } -._icon-backbone:before { background-position: 0 -3rem; } -._icon-node:before { background-position: -1rem -3rem; } -._icon-sass:before { background-position: -2rem -3rem; } -._icon-less:before { background-position: -3rem -3rem; } -._icon-angular:before { background-position: -4rem -3rem; } -._icon-coffeescript:before { background-position: 0 -4rem; } -._icon-ember:before { background-position: -1rem -4rem; } -%icon-menu { background-position: -2rem -4rem; } -%icon-home { background-position: -3rem -4rem; } -._icon-jqueryui:before { background-position: -4rem -4rem; } -._icon-jquerymobile:before { background-position: 0 -5rem; } -._icon-lodash:before { background-position: -1rem -5rem; } -._icon-php:before { background-position: -2rem -5rem; } -._icon-ruby:before { background-position: -3rem -5rem; } -._icon-rails:before { background-position: -4rem -5rem; } -._icon-python:before { background-position: 0 -6rem; } -._icon-python2:before { background-position: 0 -6rem; } -._icon-git:before { background-position: -1rem -6rem; } -._icon-redis:before { background-position: -2rem -6rem; } -._icon-postgresql:before { background-position: -3rem -6rem; } -._icon-d3:before { background-position: -4rem -6rem; } -._icon-knockout:before { background-position: 0 -7rem; } -._icon-moment:before { background-position: -1rem -7rem; } -._icon-c:before { background-position: -2rem -7rem; } -%icon-path { background-position: -3rem -7rem; } -._icon-yii:before { background-position: -4rem -7rem; } -._icon-cpp:before { background-position: 0 -8rem; } -._icon-go:before { background-position: -1rem -8rem; } -._icon-express:before { background-position: -2rem -8rem; } -._icon-grunt:before { background-position: -3rem -8rem; } -._icon-maxcdn:before { background-position: -4rem -8rem; } -._icon-laravel:before { background-position: 0 -9rem; } -._icon-haskell:before { background-position: -1rem -9rem; } -._icon-requirejs:before { background-position: -2rem -9rem; } -._icon-chai:before { background-position: -3rem -9rem; } -._icon-sinon:before { background-position: -4rem -9rem; } -._icon-cordova:before { background-position: 0 -10rem; } -._icon-markdown:before { background-position: -1rem -10rem; } -._icon-django:before { background-position: -2rem -10rem; } -._icon-xpath:before { background-position: -3rem -10rem; } -._icon-nginx:before { background-position: -4rem -10rem; } -._icon-svg:before { background-position: 0 -11rem; } -._icon-marionette:before { background-position: -1rem -11rem; } -._icon-mongoose:before { background-position: -2rem -11rem; } -._icon-phpunit:before { background-position: -3rem -11rem; } -._icon-nokogiri:before { background-position: -4rem -11rem; } -._icon-rethinkdb:before { background-position: 0 -12rem; } -._icon-react:before { background-position: -1rem -12rem; } -._icon-socketio:before { background-position: -2rem -12rem; } -._icon-modernizr:before { background-position: -3rem -12rem; } -._icon-bower:before { background-position: -4rem -12rem; } +%icon-settings { background-position: -5rem 0; } +%icon-check { background-position: -6rem 0; } +._icon-http:before { background-position: -7rem 0; @extend %darkIconFix !optional; } +._icon-jquery:before { background-position: -8rem 0; @extend %darkIconFix !optional; } +._icon-underscore:before { background-position: -9rem 0; @extend %darkIconFix !optional; } +._icon-html:before { background-position: 0 -1rem; } +._icon-css:before { background-position: -1rem -1rem; } +._icon-dom:before { background-position: -2rem -1rem; } +._icon-dom_events:before { background-position: -3rem -1rem; } +._icon-javascript:before { background-position: -4rem -1rem; } +._icon-backbone:before { background-position: -5rem -1rem; @extend %darkIconFix !optional; } +._icon-node:before { background-position: -6rem -1rem; } +._icon-sass:before { background-position: -7rem -1rem; } +._icon-less:before { background-position: -8rem -1rem; } +._icon-angular:before { background-position: -9rem -1rem; } +._icon-coffeescript:before { background-position: 0 -2rem; @extend %darkIconFix !optional; } +._icon-ember:before { background-position: -1rem -2rem; } +%icon-menu { background-position: -2rem -2rem; } +%icon-home { background-position: -3rem -2rem; } +._icon-jqueryui:before { background-position: -4rem -2rem; } +._icon-jquerymobile:before { background-position: -5rem -2rem; } +._icon-lodash:before { background-position: -6rem -2rem; } +._icon-php:before { background-position: -7rem -2rem; } +._icon-ruby:before, +._icon-minitest:before { background-position: -8rem -2rem; } +._icon-rails:before { background-position: -9rem -2rem; } +._icon-python:before, +._icon-python2:before { background-position: 0 -3rem; } +._icon-git:before { background-position: -1rem -3rem; } +._icon-redis:before { background-position: -2rem -3rem; } +._icon-postgresql:before { background-position: -3rem -3rem; } +._icon-d3:before { background-position: -4rem -3rem; } +._icon-knockout:before { background-position: -5rem -3rem; } +._icon-moment:before { background-position: -6rem -3rem; @extend %darkIconFix !optional; } +._icon-c:before { background-position: -7rem -3rem; } +%icon-path { background-position: -8rem -3rem; } +._icon-yii:before, +._icon-yii1:before { background-position: -9rem -3rem; } +._icon-cpp:before { background-position: 0 -4rem; } +._icon-go:before { background-position: -1rem -4rem; } +._icon-express:before { background-position: -2rem -4rem; } +._icon-grunt:before { background-position: -3rem -4rem; } + +._icon-laravel:before { background-position: -5rem -4rem; } +._icon-haskell:before { background-position: -6rem -4rem; } +._icon-requirejs:before { background-position: -7rem -4rem; } +._icon-chai:before { background-position: -8rem -4rem; } +._icon-sinon:before { background-position: -9rem -4rem; @extend %darkIconFix !optional; } +._icon-cordova:before { background-position: 0 -5rem; } +._icon-markdown:before { background-position: -1rem -5rem; @extend %darkIconFix !optional; } +._icon-django:before { background-position: -2rem -5rem; } +._icon-xpath:before { background-position: -3rem -5rem; } +._icon-nginx:before { background-position: -4rem -5rem; } +._icon-svg:before { background-position: -5rem -5rem; } +._icon-marionette:before { background-position: -6rem -5rem; } +._icon-mongoose:before { background-position: -7rem -5rem; } +._icon-phpunit:before { background-position: -8rem -5rem; } +._icon-nokogiri:before { background-position: -9rem -5rem; @extend %darkIconFix !optional; } +._icon-rethinkdb:before { background-position: 0 -6rem; } +._icon-react:before { background-position: -1rem -6rem; } +._icon-socketio:before { background-position: -2rem -6rem; } +._icon-modernizr:before { background-position: -3rem -6rem; } +._icon-bower:before { background-position: -4rem -6rem; } +%icon-search-white { background-position: -5rem -6rem; } +%icon-dir-white { background-position: -6rem -6rem; } +%icon-link-white { background-position: -7.25rem -6.25rem; } +%icon-settings-white { background-position: -8rem -6rem; } +%icon-check-white { background-position: -9rem -6rem; } +%icon-light { background-position: 0 -7rem; } +%icon-light-white { background-position: -1rem -7rem; } +._icon-iojs:before { background-position: -2rem -7rem; @extend %darkIconFix !optional; } +._icon-lua:before { background-position: -3rem -7rem; @extend %darkIconFix !optional; } +._icon-clojure:before { background-position: -4rem -7rem; } +._icon-symfony:before { background-position: -5rem -7rem; } +._icon-mocha:before { background-position: -6rem -7rem; } +._icon-meteor:before { background-position: -7rem -7rem; @extend %darkIconFix !optional; } +._icon-npm:before { background-position: -8rem -7rem; } +._icon-apache_http_server:before { background-position: -9rem -7rem; } diff --git a/assets/stylesheets/global/_variables.scss b/assets/stylesheets/global/_variables.scss index a2fc51b3b8..453bfcb6ea 100644 --- a/assets/stylesheets/global/_variables.scss +++ b/assets/stylesheets/global/_variables.scss @@ -1,5 +1,7 @@ $baseFont: 'Open Sans', Helvetica, Arial, sans-serif; -$monoFont: 'Source Code Pro', 'Inconsolata-g', Consolas, Menlo, monospace; +$monoFont: Consolas, 'Liberation Mono', Menlo, Courier, monospace; + +$style: 'light'; $headerHeight: 3rem; $sidebarWidth: 18rem; diff --git a/assets/stylesheets/global/variables-dark.scss b/assets/stylesheets/global/variables-dark.scss new file mode 100644 index 0000000000..53a589b1a7 --- /dev/null +++ b/assets/stylesheets/global/variables-dark.scss @@ -0,0 +1,82 @@ +$baseFont: 'Open Sans', Helvetica, Arial, sans-serif; +$monoFont: Consolas, 'Liberation Mono', Menlo, Courier, monospace; + +$style: 'dark'; + +$headerHeight: 3rem; +$sidebarWidth: 18rem; +$sidebarMediumWidth: 16rem; + +$contentBackground: #303030; + +$textColor: #c9c9c9; +$textColorLight: #858585; +$textColorLighter: #706f6f; + +$inputFocusBorder: #444; + +$focusBackground: #3d3d3e; +$focusBorder: #3d3d3e; +$focusText: #f7f2f2; + +$loadingText: #616161; +$splashText: #616161; +$promoText: #616161; + +$selectionBackground: #346fce; +$selectionBorder: #346fce; +$selectionText: #fff; + +$highlightBackground: #3d4449; + +$linkColor: #fff; +$linkColorHover: #fff; + +$headerBackground: #1c1c1c; +$headerBorder: #1c1c1c; + +$sidebarBackground: #262626; +$sidebarBorder: #262626; + +$scrollbarColor: #515151; +$scrollbarColorHover: #888; + +$pathBackground: $headerBackground; +$pathBorder: $headerBorder; + +$noticeBackground: #1c1c1c; +$noticeBorder: #1c1c1c; + +$boxBackground: #262626; +$boxBorder: #000; +$boxBorderLight: #000; +$boxHeaderBackground: #262626; + +$noteBackground: #3a4031; +$noteBorder: #000; + +$noteGreenBackground: #223e23; +$noteGreenBorder: #000; + +$noteBlueBackground: #223546; +$noteBlueBorder: #000; + +$noteOrangeBackground: #523121; +$noteOrangeBorder: #000; + +$noteRedBackground: #5b1f22; +$noteRedBorder: #000; + +$labelBackground: #3c3c3e; + +$notifBackground: rgba(#000, .85); +$notifColor: #fff; +$notifColorLight: #bbb; + +$mediumScreen: '(max-width: 800px)'; + +$contentZ: 1; +$sidebarZ: 2; +$headerZ: 3; +$noticeZ: 4; +$hoverZ: 5; diff --git a/assets/stylesheets/pages/.sass-cache/3a3912d3f706c7716d60ead2be49a2a2c584152d/_angular.scssc b/assets/stylesheets/pages/.sass-cache/3a3912d3f706c7716d60ead2be49a2a2c584152d/_angular.scssc new file mode 100644 index 0000000000..7f920d392c Binary files /dev/null and b/assets/stylesheets/pages/.sass-cache/3a3912d3f706c7716d60ead2be49a2a2c584152d/_angular.scssc differ diff --git a/assets/stylesheets/pages/_apache.scss b/assets/stylesheets/pages/_apache.scss new file mode 100644 index 0000000000..ff494bb52e --- /dev/null +++ b/assets/stylesheets/pages/_apache.scss @@ -0,0 +1,6 @@ +._apache { + @extend %simple; + + .note, .warning { @extend %note; } + .warning { @extend %note-red; } +} diff --git a/assets/stylesheets/pages/_base.scss b/assets/stylesheets/pages/_base.scss new file mode 100644 index 0000000000..59bfaf482c --- /dev/null +++ b/assets/stylesheets/pages/_base.scss @@ -0,0 +1,30 @@ +%simple { + padding-left: 1rem; + + h1, h2, h3 { margin-left: -1rem; } + h2 { @extend %block-heading; } + h3 { @extend %block-label, %label-blue; } + h4 { font-size: inherit; } + + ._mobile & { + padding-left: 0; + + h1, h2, h3 { margin-left: 0; } + } + + p > code, li > code { @extend %label; } + blockquote { @extend %note; } +} + +._cordova, +._grunt, +._less, +._lodash, +._marionette, +._markdown, +._mocha, +._mongoose, +._sinon { + @extend %simple; +} + diff --git a/assets/stylesheets/pages/_chai.scss b/assets/stylesheets/pages/_chai.scss index a74567e8ef..19c24ab594 100644 --- a/assets/stylesheets/pages/_chai.scss +++ b/assets/stylesheets/pages/_chai.scss @@ -1,13 +1,5 @@ ._chai { - padding-left: 1rem; - - > h1, > h2, h3 { margin-left: -1rem; } - > h2 { @extend %block-heading; } - h3 { @extend %block-label, %label-blue; } - h4 { font-size: 1em; } - - code { @extend %label; } - blockquote { @extend %note; } + @extend %simple; li.tag span { margin-right: 5px; } } diff --git a/assets/stylesheets/pages/_clojure.scss b/assets/stylesheets/pages/_clojure.scss new file mode 100644 index 0000000000..9f5e5db45a --- /dev/null +++ b/assets/stylesheets/pages/_clojure.scss @@ -0,0 +1,10 @@ +._clojure { + h2:not([id]) { @extend %block-heading; } + h2[id], h3 { @extend %block-label, %label-blue; } + + .type { + float: right; + font-size: .9em; + color: $textColorLight; + } +} diff --git a/assets/stylesheets/pages/_coffeescript.scss b/assets/stylesheets/pages/_coffeescript.scss index 99f7a5ef08..af4125fd76 100644 --- a/assets/stylesheets/pages/_coffeescript.scss +++ b/assets/stylesheets/pages/_coffeescript.scss @@ -1,10 +1,5 @@ ._coffeescript { - padding-left: 1rem; - - > h1, > h2 { margin-left: -1rem; } - > h2 { @extend %block-heading; } - - code { @extend %label; } + @extend %simple; // CoffeeScript / JavaScript code blocks > .code { diff --git a/assets/stylesheets/pages/_d3.scss b/assets/stylesheets/pages/_d3.scss index 0d34d1d97a..1843e84829 100644 --- a/assets/stylesheets/pages/_d3.scss +++ b/assets/stylesheets/pages/_d3.scss @@ -1,6 +1,6 @@ ._d3 { - > h2 { @extend %block-heading; } - > h3, > h4 { font-size: 1rem; } - > h6 { @extend %block-label, %label-blue; } + h2 { @extend %block-heading; } + h3, h4 { font-size: 1rem; } + h6 { @extend %block-label, %label-blue; } code { @extend %label; } } diff --git a/assets/stylesheets/pages/_express.scss b/assets/stylesheets/pages/_express.scss index 9b388a9d9f..de1bf21ec5 100644 --- a/assets/stylesheets/pages/_express.scss +++ b/assets/stylesheets/pages/_express.scss @@ -1,8 +1,6 @@ ._express { - padding-left: 1rem; + @extend %simple; - h1, h2, h3 { margin-left: -1rem; } - h2 { @extend %block-heading; } - h3 { @extend %block-label, %label-blue; } - p > code { @extend %label; } + .doc-box { @extend %note; } + .doc-warn { @extend %note-red; } } diff --git a/assets/stylesheets/pages/_go.scss b/assets/stylesheets/pages/_go.scss index 08c19bf929..218fa0df5d 100644 --- a/assets/stylesheets/pages/_go.scss +++ b/assets/stylesheets/pages/_go.scss @@ -1,7 +1,5 @@ ._go { - padding-left: 1rem; + @extend %simple; - h1, h2, #short-nav, table.dir { margin-left: -1rem; } - h2 { @extend %block-heading; } - h3 { @extend %block-label, %label-blue; } + #short-nav, table.dir { margin-left: -1rem; } } diff --git a/assets/stylesheets/pages/_lua.scss b/assets/stylesheets/pages/_lua.scss new file mode 100644 index 0000000000..68cca61e54 --- /dev/null +++ b/assets/stylesheets/pages/_lua.scss @@ -0,0 +1,5 @@ +._lua { + @extend %simple; + + .apii { float: right; } +} diff --git a/assets/stylesheets/pages/_mdn.scss b/assets/stylesheets/pages/_mdn.scss index cc103cae12..c640ddf604 100644 --- a/assets/stylesheets/pages/_mdn.scss +++ b/assets/stylesheets/pages/_mdn.scss @@ -4,7 +4,7 @@ -moz-columns: 16em; columns: 16em; - > span { + span { display: block; font-size: 1rem; font-weight: bold; diff --git a/assets/stylesheets/pages/_meteor.scss b/assets/stylesheets/pages/_meteor.scss new file mode 100644 index 0000000000..0de5a9bb36 --- /dev/null +++ b/assets/stylesheets/pages/_meteor.scss @@ -0,0 +1,15 @@ +._meteor { + @extend %simple; + + .note, .warning { @extend %note; } + .warning { @extend %note-red; } + + dl.args { margin-left: 1rem; } + + .locus, .src-code { float: right; } + .locus, .type, .src-code { + margin-left: .5em; + font-size: .9em; + } + .locus, .type { color: $textColorLight; } +} diff --git a/assets/stylesheets/pages/_moment.scss b/assets/stylesheets/pages/_moment.scss index eb5eb39928..17d39c1313 100644 --- a/assets/stylesheets/pages/_moment.scss +++ b/assets/stylesheets/pages/_moment.scss @@ -1,7 +1,7 @@ ._moment { - > h2 { @extend %block-heading; } - > h3 { @extend %block-label, %label-blue; } - > h3 > span { float: right;} + h2 { @extend %block-heading; } + h3 { @extend %block-label, %label-blue; } + h3 span { float: right;} h4 { font-size: 1em; } code { @extend %label; } } diff --git a/assets/stylesheets/pages/_nginx.scss b/assets/stylesheets/pages/_nginx.scss index 327811e840..57cdba9760 100644 --- a/assets/stylesheets/pages/_nginx.scss +++ b/assets/stylesheets/pages/_nginx.scss @@ -2,4 +2,5 @@ h4 { @extend %block-heading; } .note { @extend %note; } .directive { margin: 2.5em 0 1em; } + td > pre { margin: 0; } } diff --git a/assets/stylesheets/pages/_npm.scss b/assets/stylesheets/pages/_npm.scss new file mode 100644 index 0000000000..9bbcc74c5c --- /dev/null +++ b/assets/stylesheets/pages/_npm.scss @@ -0,0 +1,15 @@ +._npm { + @extend %simple; + + .pageColumns { + padding-left: 0; + list-style: none; + } + + .faint.heading { + font-size: .9em; + color: $textColorLight; + } + + .youtube-video iframe { width: 420px; height: 315px; } +} diff --git a/assets/stylesheets/pages/_php.scss b/assets/stylesheets/pages/_php.scss index 7ea8d4d45f..4386bd570e 100644 --- a/assets/stylesheets/pages/_php.scss +++ b/assets/stylesheets/pages/_php.scss @@ -23,8 +23,6 @@ .classsynopsis > .methodsynopsis, .classsynopsis > .fieldsynopsis { margin-left: 1em; } - .phpcode > pre { white-space: normal; } - blockquote.note { @extend %note; } blockquote.note > p { margin-bottom: 0; } diff --git a/assets/stylesheets/pages/_rdoc.scss b/assets/stylesheets/pages/_rdoc.scss index ae18b323c8..550c69d174 100644 --- a/assets/stylesheets/pages/_rdoc.scss +++ b/assets/stylesheets/pages/_rdoc.scss @@ -1,11 +1,11 @@ ._rdoc { - > .description, > .documentation-section { padding-left: 1rem; } - > .description > h2, header > h3, > h2 { @extend %block-heading; } - > .description > h2, header > h3, .method-heading { margin-left: -1rem; } + .description, > .documentation-section { padding-left: 1rem; } + .description > h2, header > h3, > h2 { @extend %block-heading; } + .description > h2, header > h3, .method-heading { margin-left: -1rem; } .description > h1 { font-size: 1rem; } .method-description > h2, h3, h4, h5, h6 { font-size: 1em; } - .method-heading { + .method-heading, .signature { font-weight: bold; @extend %block-label, %label-blue; diff --git a/assets/stylesheets/pages/_requirejs.scss b/assets/stylesheets/pages/_requirejs.scss index d186cca4fe..34fddb4cad 100644 --- a/assets/stylesheets/pages/_requirejs.scss +++ b/assets/stylesheets/pages/_requirejs.scss @@ -1,10 +1,5 @@ ._requirejs { - padding-left: 1rem; - - h1, h2, h3, ul.index { margin-left: -1rem; } - h2 { @extend %block-heading; } - h3 { @extend %block-label, %label-blue; } - h4 { font-size: 1em; } + @extend %simple; ul.index ul { margin: .5em 0; } } diff --git a/assets/stylesheets/pages/_rfc.scss b/assets/stylesheets/pages/_rfc.scss index ac2bd42d9d..b522b30890 100644 --- a/assets/stylesheets/pages/_rfc.scss +++ b/assets/stylesheets/pages/_rfc.scss @@ -1,6 +1,15 @@ -._rfc { - padding-left: 1rem; +._rfc-pre { + font-size: .8125rem; + min-width: 38rem; + @extend %code; - > h1, > h2 { margin-left: -1rem; } > h2 { @extend %block-heading; } + > h3 { @extend %block-label, %label-blue; } + > h4 { @extend %block-label; } + > h3, > h4 { font-size: .875rem; } + + > h1, > h2, > h3, > h4, > h5 { + margin: 0; + font-family: $baseFont; + } } diff --git a/assets/stylesheets/pages/_sphinx.scss b/assets/stylesheets/pages/_sphinx.scss index 44b9f1f3a9..5c87eae3fb 100644 --- a/assets/stylesheets/pages/_sphinx.scss +++ b/assets/stylesheets/pages/_sphinx.scss @@ -1,6 +1,6 @@ ._sphinx { h2, h3 { @extend %block-heading; } - dl:not(.docutils) > dt { @extend %block-label, %label-blue; } + dl:not(.docutils) dt { @extend %block-label, %label-blue; } dt + dt { margin-top: -.5em; } .note, .admonition, .versionadded, .versionchanged, .deprecated-removed { @extend %note; } diff --git a/file_scraper_docs/.download_index b/file_scraper_docs/.download_index new file mode 100644 index 0000000000..fa7c1ad59a --- /dev/null +++ b/file_scraper_docs/.download_index @@ -0,0 +1,4 @@ +C & c++: http://en.cppreference.com/w/Cppreference:Archives +python 2.7: https://docs.python.org/2.7/download.html +python 3: https://docs.python.org/3/download.html +django: https://docs.djangoproject.com/en/1.8/ \ No newline at end of file diff --git a/lib/app.rb b/lib/app.rb index 05e103892e..19977b6e97 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -4,6 +4,7 @@ class App < Sinatra::Application Bundler.require environment require 'sinatra/cookies' + require 'tilt/erubis' Rack::Mime::MIME_TYPES['.webapp'] = 'application/x-web-app-manifest+json' @@ -17,7 +18,7 @@ class App < Sinatra::Application set :assets_prefix, 'assets' set :assets_path, -> { File.join(public_folder, assets_prefix) } set :assets_manifest_path, -> { File.join(assets_path, 'manifest.json') } - set :assets_compile, %w(*.png docs.js application.js application.css) + set :assets_compile, %w(*.png docs.js docs.json application.js application.css application-dark.css) require 'yajl/json_gem' set :docs_prefix, 'docs' @@ -25,6 +26,7 @@ class App < Sinatra::Application set :docs_path, -> { File.join(public_folder, docs_prefix) } set :docs_manifest_path, -> { File.join(docs_path, 'docs.json') } set :docs, -> { Hash[JSON.parse(File.read(docs_manifest_path)).map! { |doc| [doc['slug'], doc] }] } + set :default_docs, %w(css dom dom_events html http javascript) set :news_path, -> { File.join(root, assets_prefix, 'javascripts', 'news.json') } set :news, -> { JSON.parse(File.read(news_path)) } @@ -37,6 +39,7 @@ class App < Sinatra::Application config.environment = sprockets config.prefix = "/#{assets_prefix}" config.public_path = public_folder + config.protocol = :relative end end @@ -56,14 +59,14 @@ class App < Sinatra::Application configure :production do set :static, false - set :docs_host, 'http://docs.devdocs.io' + set :docs_host, '//docs.devdocs.io' use Rack::ConditionalGet use Rack::ETag use Rack::Deflater use Rack::Static, root: 'public', - urls: %w(/assets /docs /images /favicon.ico /robots.txt /opensearch.xml /manifest.webapp), + urls: %w(/assets /docs/ /images /favicon.ico /robots.txt /opensearch.xml /manifest.webapp), header_rules: [ [:all, {'Cache-Control' => 'no-cache, max-age=0'}], ['/assets', {'Cache-Control' => 'public, max-age=604800'}], @@ -75,11 +78,15 @@ class App < Sinatra::Application Sprockets::Helpers.configure do |config| config.digest = true - config.asset_host = 'maxcdn.devdocs.io' + config.asset_host = 'cdn.devdocs.io' config.manifest = Sprockets::Manifest.new(sprockets, assets_manifest_path) end end + configure :test do + set :docs_manifest_path, -> { File.join(root, 'test', 'files', 'docs.json') } + end + helpers do include Sinatra::Cookies include Sprockets::Helpers @@ -88,19 +95,29 @@ def browser @browser ||= Browser.new ua: request.user_agent end + UNSUPPORTED_IE_VERSIONS = %w(6 7 8 9).freeze + def unsupported_browser? - browser.ie? && %w(6 7 8 9).include?(browser.version) + browser.ie? && UNSUPPORTED_IE_VERSIONS.include?(browser.version) end - def doc_index_urls - cookie = cookies[:docs] - return [] if cookie.nil? || cookie.empty? + def docs + @docs ||= begin + cookie = cookies[:docs] - cookie.split('/').inject [] do |result, slug| + if cookie.nil? || cookie.empty? + settings.default_docs + else + cookie.split('/') + end + end + end + + def doc_index_urls + docs.each_with_object [] do |slug, result| if doc = settings.docs[slug] result << File.join('', settings.docs_prefix, doc['index_path']) + "?#{doc['mtime']}" end - result end end @@ -111,12 +128,61 @@ def doc_index_page? def query_string_for_redirection request.query_string.empty? ? nil : "?#{request.query_string}" end + + def main_stylesheet_path + stylesheet_paths[dark_theme? ? :dark : :default] + end + + def alternate_stylesheet_path + stylesheet_paths[dark_theme? ? :default : :dark] + end + + def stylesheet_paths + @stylesheet_paths ||= { + default: stylesheet_path('application'), + dark: stylesheet_path('application-dark') + } + end + + def app_size + @app_size ||= cookies[:size].nil? ? '18rem' : "#{cookies[:size]}px" + end + + def app_layout + cookies[:layout] + end + + def app_theme + @app_theme ||= cookies[:dark].nil? ? 'default' : 'dark' + end + + def dark_theme? + app_theme == 'dark' + end + + def redirect_via_js(path) # courtesy of HTML5 App Cache + response.set_cookie :initial_path, value: path, expires: Time.now + 15, path: '/' + redirect '/', 302 + end + + def supports_js_redirection? + browser.modern? && !cookies.empty? + end end before do halt erb :unsupported if unsupported_browser? end + OUT_HOST = 'out.devdocs.io'.freeze + + before do + if request.host == OUT_HOST && !request.path.start_with?('/s/') + query_string = "?#{request.query_string}" unless request.query_string.empty? + redirect "http://devdocs.io#{request.path}#{query_string}", 302 + end + end + get '/manifest.appcache' do content_type 'text/cache-manifest' expires 0, :'no-cache' @@ -124,13 +190,17 @@ def query_string_for_redirection end get '/' do - return redirect '/' unless request.query_string.empty? + return redirect '/' unless request.query_string.empty? # courtesy of HTML5 App Cache erb :index end %w(offline about news help).each do |page| get "/#{page}" do - redirect "/#/#{page}", 302 + if supports_js_redirection? + redirect_via_js "/#{page}" + else + redirect "/#/#{page}", 302 + end end end @@ -142,12 +212,32 @@ def query_string_for_redirection 200 end - get '/s/maxcdn' do - redirect 'https://www.maxcdn.com/?utm_source=devdocs&utm_medium=banner&utm_campaign=devdocs' + %w(docs.json application.js application.css).each do |asset| + class_eval <<-CODE, __FILE__, __LINE__ + 1 + get '/#{asset}' do + redirect asset_path('#{asset}', protocol: 'http') + end + CODE end - get '/s/shopify' do - redirect 'http://www.shopify.com/careers?utm_source=devdocs&utm_medium=banner&utm_campaign=devdocs' + { + '/s/maxcdn' => 'https://www.maxcdn.com/?utm_source=devdocs&utm_medium=banner&utm_campaign=devdocs', + '/s/shopify' => 'https://www.shopify.com/careers?utm_source=devdocs&utm_medium=banner&utm_campaign=devdocs', + '/s/jetbrains' => 'https://www.jetbrains.com/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs', + '/s/jetbrains/ruby' => 'https://www.jetbrains.com/ruby/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs', + '/s/jetbrains/python' => 'https://www.jetbrains.com/pycharm/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs', + '/s/jetbrains/c' => 'https://www.jetbrains.com/clion/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs', + '/s/jetbrains/web' => 'https://www.jetbrains.com/webstorm/?utm_source=devdocs&utm_medium=sponsorship&utm_campaign=devdocs', + '/s/code-school' => 'http://www.codeschool.com/?utm_campaign=devdocs&utm_content=homepage&utm_source=devdocs&utm_medium=sponsorship', + '/s/tw' => 'https://twitter.com/intent/tweet?url=http%3A%2F%2Fdevdocs.io&via=DevDocs&text=All-in-one%2C%20offline%20API%20documentation%20browser%3A', + '/s/fb' => 'https://twitter.com/intent/tweet?url=http%3A%2F%2Fdevdocs.io&via=DevDocs&text=All-in-one%2C%20offline%20API%20documentation%20browser%3A', + '/s/re' => 'http://www.reddit.com/submit?url=http%3A%2F%2Fdevdocs.io&title=All-in-one%2C%20offline%20API%20documentation%20browser&resubmit=true' + }.each do |path, url| + class_eval <<-CODE, __FILE__, __LINE__ + 1 + get '#{path}' do + redirect '#{url}' + end + CODE end get %r{\A/feed(?:\.atom)?\z} do @@ -155,18 +245,6 @@ def query_string_for_redirection settings.news_feed end - get '/s/tw' do - redirect 'https://twitter.com/intent/tweet?url=http%3A%2F%2Fdevdocs.io&via=DevDocs&text=All-in-one%2C%20quickly%20searchable%20API%20docs%3A' - end - - get '/s/fb' do - redirect 'https://www.facebook.com/sharer/sharer.php?u=http%3A%2F%2Fdevdocs.io' - end - - get '/s/re' do - redirect 'http://www.reddit.com/submit?url=http%3A%2F%2Fdevdocs.io&title=All-in-one%2C%20quickly%20searchable%20API%20docs&resubmit=true' - end - get %r{\A/(\w+)(\-[\w\-]+)?(/.*)?\z} do |doc, type, rest| return 404 unless @doc = settings.docs[doc] @@ -174,6 +252,8 @@ def query_string_for_redirection redirect "/#{doc}#{type}/#{query_string_for_redirection}" elsif rest.length > 1 && rest.end_with?('/') redirect "/#{doc}#{type}#{rest[0...-1]}#{query_string_for_redirection}" + elsif docs.include?(doc) && supports_js_redirection? + redirect_via_js(request.path) else erb :other end diff --git a/lib/docs/core/doc.rb b/lib/docs/core/doc.rb index 6388c56804..1df17a88e4 100644 --- a/lib/docs/core/doc.rb +++ b/lib/docs/core/doc.rb @@ -4,18 +4,60 @@ class Doc DB_FILENAME = 'db.json' class << self - attr_accessor :name, :slug, :type, :version, :abstract + include Instrumentable + + attr_accessor :name, :slug, :type, :release, :abstract, :links def inherited(subclass) subclass.type = type end + def version(version = nil, &block) + return @version unless block_given? + + klass = Class.new(self) + klass.name = name + klass.slug = slug + klass.version = version + klass.release = release + klass.links = links + klass.class_exec(&block) + @versions ||= [] + @versions << klass + klass + end + + def version=(value) + @version = value.to_s + end + + def versions + @versions.presence || [self] + end + + def version? + version.present? + end + + def versioned? + @versions.presence + end + def name @name || super.try(:demodulize) end def slug - @slug || name.try(:downcase) + slug = @slug || name.try(:downcase) + version? ? "#{slug}~#{version_slug}" : slug + end + + def version_slug + slug = version.downcase + slug.gsub! '+', 'p' + slug.gsub! '#', 's' + slug.gsub! %r{[^a-z0-9\_\.]}, '_' + slug end def path @@ -31,12 +73,11 @@ def db_path end def as_json - { name: name, - slug: slug, - type: type, - version: version, - index_path: index_path, - db_path: db_path } + json = { name: name, slug: slug, type: type } + json[:links] = links if links.present? + json[:version] = version if version.present? || defined?(@version) + json[:release] = release if release.present? + json end def store_page(store, id) @@ -63,8 +104,8 @@ def store_pages(store) end if index.present? - store.write INDEX_FILENAME, index.to_json - store.write DB_FILENAME, pages.to_json + store_index(store, INDEX_FILENAME, index) + store_index(store, DB_FILENAME, pages) true else false @@ -77,6 +118,13 @@ def store_pages(store) def store_page?(page) page[:entries].present? end + + def store_index(store, filename, index) + old_json = store.read(filename) || '{}' + new_json = index.to_json + instrument "#{filename.remove('.json')}.doc", before: old_json, after: new_json + store.write(filename, new_json) + end end def initialize diff --git a/lib/docs/core/entry_index.rb b/lib/docs/core/entry_index.rb index 3c819c70b8..39df9b7092 100644 --- a/lib/docs/core/entry_index.rb +++ b/lib/docs/core/entry_index.rb @@ -1,21 +1,22 @@ require 'yajl/json_gem' module Docs - class EntryIndex + class EntryIndex < ReflyEntryIndex attr_reader :entries, :types def initialize @entries = [] + @index = Set.new @types = Hash.new { |hash, key| hash[key] = Type.new key } end - def add(entry) - if entry.is_a? Array - entry.each(&method(:add)) - else - add_entry(entry) unless entry.root? - end - end + # def add(entry) + # if entry.is_a? Array + # entry.each(&method(:add)) + # else + # add_entry(entry) unless entry.root? + # end + # end def empty? @entries.empty? @@ -38,8 +39,10 @@ def to_json private def add_entry(entry) - @entries << entry.dup - @types[entry.type].count += 1 if entry.type + if @index.add?(entry.as_json.to_s) + @entries << entry.dup + @types[entry.type].count += 1 if entry.type + end end def entries_as_json diff --git a/lib/docs/core/filesmodified.txt b/lib/docs/core/filesmodified.txt new file mode 100644 index 0000000000..e64c668d27 --- /dev/null +++ b/lib/docs/core/filesmodified.txt @@ -0,0 +1,2 @@ +entryindex.rb +entries.rb \ No newline at end of file diff --git a/lib/docs/core/filter.rb b/lib/docs/core/filter.rb index de1765449f..6d3d4b466e 100644 --- a/lib/docs/core/filter.rb +++ b/lib/docs/core/filter.rb @@ -20,6 +20,10 @@ def base_url context[:base_url] end + def links + context[:links] + end + def current_url context[:url] end diff --git a/lib/docs/core/manifest.rb b/lib/docs/core/manifest.rb index 5a00a1022f..8680ab5d7b 100644 --- a/lib/docs/core/manifest.rb +++ b/lib/docs/core/manifest.rb @@ -14,9 +14,11 @@ def store end def as_json - indexed_docs.map(&:as_json).each do |json| - json[:mtime] = doc_mtime(json) - json[:db_size] = doc_db_size(json) + indexed_docs.map do |doc| + json = doc.as_json + json[:mtime] = doc_mtime(doc) + json[:db_size] = doc_db_size(doc) + json end end @@ -33,11 +35,11 @@ def indexed_docs end def doc_mtime(doc) - [@store.mtime(doc[:index_path]).to_i, @store.mtime(doc[:db_path]).to_i].max + [@store.mtime(doc.index_path).to_i, @store.mtime(doc.db_path).to_i].max end def doc_db_size(doc) - @store.size(doc[:db_path]) + @store.size(doc.db_path) end end end diff --git a/lib/docs/core/models/refly_entry.rb b/lib/docs/core/models/refly_entry.rb new file mode 100644 index 0000000000..340f40b1cb --- /dev/null +++ b/lib/docs/core/models/refly_entry.rb @@ -0,0 +1,63 @@ +module Docs + class ReflyEntry < Docs::Entry + attr_accessor :name, :type, :path, :parsed_uri,:anchor, :parent_uri, :docset, :source_url + def initialize(name = nil, path = nil, type = nil, parsed_uri = nil, anchor = nil, parent_uri = nil, docset = nil, source_url = nil) + self.name = name + self.path = path + self.type = type + self.parsed_uri = parsed_uri + self.anchor = anchor + self.parent_uri = parent_uri + self.docset = docset + self.source_url = source_url + end + + def ==(other) + other.name == name && other.path == path && other.type == type + end + + def <=>(other) + name.to_s.casecmp(other.name.to_s) + end + + def docset=(value) + @docset = value.try :strip + end + + def parsed_uri=(value) + @parsed_uri = value.try :strip + end + + def anchor=(value) + @anchor = value.try :strip + end + + def parent_uri=(value) + @parent_uri = value.try :strip + end + + def source_url=(value) + @source_url = value.try :strip + end + + def name=(value) + @name = value.try :strip + end + + def type=(value) + @type = value.try :strip + end + + def root? + if anchor!='' + return false + else + return path == 'index' + end + end + + def as_json + { name: name, path: path, source_url: source_url, type: type, parsed_uri: parsed_uri, anchor: anchor, parent_uri: parent_uri, docset: docset } + end + end +end diff --git a/lib/docs/core/models/type.rb b/lib/docs/core/models/type.rb index 2460250b91..b9869ae7c3 100644 --- a/lib/docs/core/models/type.rb +++ b/lib/docs/core/models/type.rb @@ -7,8 +7,14 @@ def initialize(*args) self.count ||= 0 end + STARTS_WITH_INTEGER = /\A\d/ + def <=>(other) - name.to_s.casecmp(other.name.to_s) + if name && other && name =~ STARTS_WITH_INTEGER && other.name =~ STARTS_WITH_INTEGER + name.to_i <=> other.name.to_i + else + name.to_s.casecmp(other.name.to_s) + end end def slug diff --git a/lib/docs/core/parser.rb b/lib/docs/core/parser.rb index 43a0890428..b3bf79155c 100644 --- a/lib/docs/core/parser.rb +++ b/lib/docs/core/parser.rb @@ -11,7 +11,7 @@ def html private def document? - @content =~ /\A\s*])*>/ + return content.gsub!(re, '') + end + + def WrapPreContentWithCode(classname) + css('pre').each do |node| + node['class'] = classname + if !node.children[0] || node.children[0].name != 'code' + codetag = Nokogiri::XML::Node.new "code", @doc + codetag.children = node.children + node.children = codetag + end + end + end + + def WrapContentWithDivs(classname) + divremovable = Nokogiri::XML::Node.new "div", @doc + divcontainer = Nokogiri::XML::Node.new "div", @doc + divcontainer['class'] = classname + divcontainer.children = @doc + divremovable.children = divcontainer + @doc = divremovable + end + end +end diff --git a/lib/docs/core/refly_request.rb b/lib/docs/core/refly_request.rb new file mode 100644 index 0000000000..955b87a568 --- /dev/null +++ b/lib/docs/core/refly_request.rb @@ -0,0 +1,29 @@ +module Docs + class ReflyRequest < Typhoeus::Request + include Instrumentable + + USER_AGENTS = [ + 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6', + 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko)', + 'Chrome/6.0.472.63 Safari/534.3', + 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', + 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', + 'Yahoo! Slurp/Site Explorer', + 'Googlebot/2.1 ( http://www.googlebot.com/bot.html)', + 'msnbot/1.1 (+http://search.msn.com/msnbot.htm)', + 'Python-urllib/2.1', + 'Version/3.0 Mobile/1A543a Safari/419.3', + 'Version/6.0.0.141 Mobile Safari/534.1+', + ] + + def initialize(url, options = {}) + rand_user_agent = USER_AGENTS[rand(USER_AGENTS.length)] + default_options = { + followlocation: true, + headers: { 'User-Agent' => rand_user_agent } + } + super url.to_s, default_options.merge(options) + + end + end +end diff --git a/lib/docs/core/response.rb b/lib/docs/core/response.rb index 2908c10125..64ecc30262 100644 --- a/lib/docs/core/response.rb +++ b/lib/docs/core/response.rb @@ -4,6 +4,10 @@ def success? code == 200 end + def error? + code != 404 && code != 403 && code >= 400 && code <= 599 + end + def empty? body.empty? end diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb index 634ee8576e..2d3b43eec9 100644 --- a/lib/docs/core/scraper.rb +++ b/lib/docs/core/scraper.rb @@ -3,7 +3,7 @@ module Docs class Scraper < Doc class << self - attr_accessor :base_url, :root_path, :initial_paths, :options, :html_filters, :text_filters + attr_accessor :base_url, :root_path, :initial_paths, :options, :html_filters, :text_filters, :stubs def inherited(subclass) super @@ -13,22 +13,30 @@ def inherited(subclass) autoload_all "docs/filters/#{to_s.demodulize.underscore}", 'filter' end + subclass.base_url = base_url subclass.root_path = root_path subclass.initial_paths = initial_paths.dup subclass.options = options.deep_dup subclass.html_filters = html_filters.inheritable_copy subclass.text_filters = text_filters.inheritable_copy + subclass.stubs = stubs.dup end def filters html_filters.to_a + text_filters.to_a end + + def stub(path, &block) + @stubs[path] = block + @stubs + end end include Instrumentable self.initial_paths = [] self.options = {} + self.stubs = {} self.html_filters = FilterStack.new self.text_filters = FilterStack.new @@ -36,6 +44,23 @@ def filters html_filters.push 'container', 'clean_html', 'normalize_urls', 'internal_urls', 'normalize_paths' text_filters.push 'inner_html', 'clean_text', 'attribution' + def initialize + super + initialize_stubs + end + + def initialize_stubs + self.class.stubs.each do |path, block| + Typhoeus.stub(url_for(path)).and_return do + Typhoeus::Response.new \ + effective_url: url_for(path), + code: 200, + headers: { 'Content-Type' => 'text/html' }, + body: self.instance_exec(&block) + end + end + end + def build_page(path) response = request_one url_for(path) result = handle_response(response) @@ -100,6 +125,7 @@ def options (options[:only] ||= []).concat initial_paths + (root_path? ? [root_path] : ['', '/']) end + options.merge!(additional_options) if respond_to?(:additional_options, true) options.freeze end end @@ -134,6 +160,9 @@ def handle_response(response) else instrument 'ignore_response.scraper', response: response end + rescue => e + puts "URL: #{response.url}" + raise e end def process_response(response) @@ -149,5 +178,75 @@ def pipeline_context(response) def parse(string) Parser.new(string).html end + + def with_filters(*filters) + stack = FilterStack.new + stack.push(*filters) + pipeline.instance_variable_set :@filters, stack.to_a.freeze + yield + ensure + @pipeline = nil + end + + module FixInternalUrlsBehavior + def self.included(base) + base.extend ClassMethods + end + + module ClassMethods + attr_reader :internal_urls + + def store_pages(store) + instrument 'info.doc', msg: 'Building internal urls...' + with_internal_urls do + instrument 'info.doc', msg: 'Building pages...' + super + end + end + + private + + def with_internal_urls + @internal_urls = new.fetch_internal_urls + yield + ensure + @internal_urls = nil + end + end + + def fetch_internal_urls + result = [] + build_pages do |page| + result << base_url.subpath_to(page[:response_url]) if page[:entries].present? + end + result + end + + def initial_urls + return super unless self.class.internal_urls + @initial_urls ||= self.class.internal_urls.map(&method(:url_for)).freeze + end + + private + + def additional_options + if self.class.internal_urls + { + only: self.class.internal_urls.to_set, + only_patterns: nil, + skip: nil, + skip_patterns: nil, + skip_links: nil, + fixed_internal_urls: true + } + else + {} + end + end + + def process_response(response) + super.merge! response_url: response.url + end + end end end diff --git a/lib/docs/core/scrapers/file_scraper.rb b/lib/docs/core/scrapers/file_scraper.rb index 8c08721396..58a252da2f 100644 --- a/lib/docs/core/scrapers/file_scraper.rb +++ b/lib/docs/core/scrapers/file_scraper.rb @@ -8,6 +8,7 @@ class << self def inherited(subclass) super subclass.base_url = base_url + subclass.dir = dir end end diff --git a/lib/docs/core/scrapers/url_scraper.rb b/lib/docs/core/scrapers/url_scraper.rb index cb918b1bfa..6843628190 100644 --- a/lib/docs/core/scrapers/url_scraper.rb +++ b/lib/docs/core/scrapers/url_scraper.rb @@ -2,14 +2,17 @@ module Docs class UrlScraper < Scraper class << self attr_accessor :params + attr_accessor :headers def inherited(subclass) super subclass.params = params.deep_dup + subclass.headers = headers.deep_dup end end self.params = {} + self.headers = { 'User-Agent' => 'devdocs.io' } private @@ -22,11 +25,63 @@ def request_all(urls, &block) end def request_options - { params: self.class.params } + { params: self.class.params, headers: self.class.headers } end def process_response?(response) + if response.error? + raise "Error status code (#{response.code}): #{response.url}" + end + response.success? && response.html? && base_url.contains?(response.effective_url) end + + module FixRedirectionsBehavior + def self.included(base) + base.extend ClassMethods + end + + module ClassMethods + attr_reader :redirections + + def store_pages(store) + instrument 'info.doc', msg: 'Fetching redirections...' + with_redirections do + instrument 'info.doc', msg: 'Building pages...' + super + end + end + + private + + def with_redirections + @redirections = new.fetch_redirections + yield + ensure + @redirections = nil + end + end + + def fetch_redirections + result = {} + with_filters 'container', 'normalize_urls', 'internal_urls' do + build_pages do |page| + next if page[:response_effective_path] == page[:response_path] + result[page[:response_path].downcase] = page[:response_effective_path] + end + end + result + end + + private + + def process_response(response) + super.merge! response_effective_path: response.effective_path, response_path: response.path + end + + def additional_options + { redirections: self.class.redirections } + end + end end end diff --git a/lib/docs/db_storage/import_docset.py b/lib/docs/db_storage/import_docset.py new file mode 100755 index 0000000000..75355a7bb8 --- /dev/null +++ b/lib/docs/db_storage/import_docset.py @@ -0,0 +1,33 @@ +from importer import DocImporter +from scraper import Scraper +import sys +import time +def main(argv): + print 'process started at ' + time.strftime("%d/%m/%Y %H:%M:%S") + docset = '' + mode = '' + action = '' + connstring = '' + for arg in argv: + if arg[:2]== '-a': + action = arg[2:] + elif arg[:2]== '-d': + docset = arg[2:] + elif arg[:2]== '-m': + mode = arg[2:] + elif arg[:3]== '-cs': + connstring = arg[3:] + + if mode=='' and docset!='': + mode = 'single' + if action == 'import': + importer = DocImporter(docset, mode, connstring) + elif action == 'scrap': + scraper = Scraper(docset, 'importer.cfg') + else: + scraper = Scraper(docset,'importer.cfg') + importer = DocImporter(docset, mode, connstring) + print 'process finished at ' + time.strftime("%d/%m/%Y %H:%M:%S") + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/lib/docs/db_storage/importer.cfg b/lib/docs/db_storage/importer.cfg new file mode 100644 index 0000000000..0d2b52c688 --- /dev/null +++ b/lib/docs/db_storage/importer.cfg @@ -0,0 +1,970 @@ +[Connection] +User: postgres +Password: +Host: 127.0.0.1 +DBname: slashdb +[Path] +base_path: ../../../public/docs/ + +[Config] +debugMode: True +[angular] +name: AngularJS +parsed_name: angularjs +default_uri: /angularjs/ng +scraper_name: angular +version: 1.4.5 +zip_url: +sort: 1 +additional: false +active: true +[apache] +name: Apache +parsed_name: apache +default_uri: /apache/core +scraper_name: apache +version: 2.4.12 +zip_url: +sort: 2 +additional: false +active: true +[aspnet] +name: ASP.NET +parsed_name: asp.net +default_uri: /asp.net/index +scraper_name: +version: +zip_url: +sort: 3 +additional: false +active: false +[elixir] +name: Elixir +parsed_name: elixir +default_uri: /elixir/index +scraper_name: elixir +version: 1.1.1 +zip_url: +sort: 4 +additional: false +active: true +[backbone] +name: BackboneJS +parsed_name: backbonejs +default_uri: /backbonejs/backbone.$ +scraper_name: backbone +version: 1.2.3 +zip_url: +sort: 5 +additional: false +active: true +[bower] +name: Bower +parsed_name: bower +default_uri: /bower/creating_packages +scraper_name: bower +version: 1.4.1 +zip_url: +sort: 6 +additional: false +active: false +[bootstrap] +name: Bootstrap +parsed_name: bootstrap +default_uri: /bootstrap/getting_started +scraper_name: bootstrap +version: 3.3.35 +zip_url: +sort: 7 +additional: false +active: true +[browserify] +name: Browserify +parsed_name: browserify +default_uri: /browserify/getting_started +scraper_name: browserify +version: 11.0.1 +zip_url: +sort: 8 +additional: false +active: true +[c] +name: C +parsed_name: c +default_uri: /c/c_keywords +scraper_name: c +version: +zip_url: http://upload.cppreference.com/mwiki/images/6/6c/html_book_20141118.tar.gz +sort: 9 +additional: false +active: true +[chai] +name: Chai +parsed_name: chai +default_uri: /chai/api_reference +scraper_name: chai +version: +zip_url: +sort: 11 +additional: false +active: false +[chef] +name: Chef +parsed_name: chef +default_uri: chef/all_about_chef_... +scraper_name: chef +version: 12.4 +zip_url: +sort: 12 +additional: false +active: true +[clojure] +name: Clojure +parsed_name: clojure +default_uri: /clojure/clojure.core +scraper_name: clojure +version: 1.7 +zip_url: +sort: 13 +additional: false +active: true +[coffeescript] +name: CoffeeScript +parsed_name: coffeescript +default_uri: /coffeescript/index +scraper_name: coffeescript +version: 1.10.0 +zip_url: +sort: 14 +additional: false +active: true +[cordova] +name: Apache Cordova +parsed_name: apache_cordova +default_uri: /apache-cordova/overview +scraper_name: cordova +version: 5.1.1 +zip_url: +sort: 15 +additional: false +active: true +[cpp] +name: Cpp +parsed_name: cpp +default_uri: /cpp/c++_language +scraper_name: cpp +version: +zip_url: +sort: 16 +additional: false +active: true +[css] +name: CSS +parsed_name: css +default_uri: /css/universal_selectors +scraper_name: css +version: +zip_url: +sort: 0 +additional: false +active: true +[d3] +name: D3 +parsed_name: d3 +default_uri: /d3/namespaces +scraper_name: d3 +version: +zip_url: +sort: 17 +additional: false +active: false +[django] +name: Django +parsed_name: django +default_uri: /django/getting_started +scraper_name: django +version: 1.8.4 +zip_url: https://docs.djangoproject.com/m/docs/django-docs-1.8-en.zip +sort: 18 +additional: false +active: true +[dom] +name: DOM +parsed_name: dom +default_uri: /dom/document +scraper_name: dom +version: +zip_url: +sort: 10 +additional: false +active: true +[dojo] +name: Dojo +parsed_name: dojo +default_uri: /dojo/index +scraper_name: dojo +version: 1.10 +zip_url: +sort: 19 +additional: false +active: true +[dom_events] +name: DOM Events +parsed_name: dom_events +default_uri: /dom_events/click +scraper_name: dom_events +version: +zip_url: +sort: 20 +additional: false +active: true +[drupal] +name: Drupal +parsed_name: drupal +default_uri: /drupal +scraper_name: drupal +version: +zip_url: +sort: 21 +additional: false +active: false +[ember] +name: EmberJS +parsed_name: emberjs +default_uri: /ember/classes/namespace +scraper_name: ember +version: 2.0.1 +zip_url: +sort: 22 +additional: false +active: true +[express] +name: Express +parsed_name: express +default_uri: /express/express +scraper_name: express +version: +zip_url: +sort: 23 +additional: false +active: true +[flow] +name: Flow +parsed_name: flow +default_uri: /flow/index +scraper_name: flow +version: +zip_url: +sort: 24 +additional: false +active: false +[git] +name: Git +parsed_name: git +default_uri: /git/git +scraper_name: git +version: +zip_url: +sort: 25 +additional: false +active: false +[go] +name: Go +parsed_name: go +default_uri: /go/go_programming_language/community +scraper_name: go +version: 1.5.0 +zip_url: +sort: 26 +additional: false +active: true +[grunt] +name: Grunt +parsed_name: grunt +default_uri: /grunt/grunt +scraper_name: grunt +version: 0.4.5 +zip_url: +sort: 27 +additional: false +active: true +[html] +name: HTML +parsed_name: html +default_uri: /html/html_attribute_reference +scraper_name: html +version: +zip_url: +sort: 30 +additional: false +active: true +[http] +name: Http +parsed_name: http +default_uri: /http/200_ok +scraper_name: http +version: +zip_url: +sort: 28 +additional: false +active: true +[haskell] +name: Haskell +parsed_name: haskell +default_uri: /haskell/index +scraper_name: haskell +version: 7.10.2 +zip_url: +sort: 29 +additional: false +active: true +[iojs] +name: ioJS +parsed_name: iojs +default_uri: /iojs/index +scraper_name: iojs +version: 3.2.0 +zip_url: +sort: 31 +additional: false +active: true +[jasmine] +name: Jasmine +parsed_name: jasmine +default_uri: /jasmine/introduction +scraper_name: jasmine +version: edge +zip_url: +sort: 32 +additional: false +active: true +[javascript] +name: JavaScript +parsed_name: javascript +default_uri: /javascript/statements_and_declarations +scraper_name: javascript +version: +zip_url: +sort: 40 +additional: false +active: true +[java7se] +name: Java 7 SE +parsed_name: java7se +default_uri: /java7se/ +scraper_name: java7se +version: 7_se +zip_url: +sort: 33 +additional: false +active: true +[jest] +name: Jest +parsed_name: jest +default_uri: /jest/getting_started +scraper_name: jest +version: +zip_url: +sort: 34 +additional: false +active: false +[jquery] +name: JQuery +parsed_name: jquery +default_uri: /jquery/index +scraper_name: jquery_core +version: up_to_2.1.0 +zip_url: +sort: 35 +additional: false +active: true +[jqueryui] +name: JQuery UI +parsed_name: jquery_ui +default_uri: /jqueryui/color-animation +scraper_name: jquery_ui +version: 1.10.4 +zip_url: +sort: 36 +additional: false +active: true +[jquerymobile] +name: JQuery Mobile +parsed_name: jquery_mobile +default_uri: /jquerymobile/classes +scraper_name: jquery_mobile +version: 1.4.0 +zip_url: +sort: 37 +additional: false +active: true +[knex] +name: Knex +parsed_name: knex +default_uri: /knex/index +scraper_name: knex +version: 0.8.6 +zip_url: +sort: 38 +additional: false +active: true +[knockout] +name: KnockOut +parsed_name: knockout +default_uri: /knockout/knockout.js +scraper_name: knockout +version: 3.3.0 +zip_url: +sort: 39 +additional: false +active: true +[less] +name: Less +parsed_name: less +default_uri: /less/misc_functions +scraper_name: less +version: 2.5.0 +zip_url: +sort: 41 +additional: false +active: true +[laravel] +name: Laravel +parsed_name: laravel +default_uri: /laravel/namespaces +scraper_name: laravel +version: 5.1.1 +zip_url: +sort: 42 +additional: false +active: false +[lodash] +name: Lo-Dash +parsed_name: lodash +default_uri: /lodash/_ +scraper_name: lodash +version: 3.10.0 +zip_url: +sort: 43 +additional: false +active: true +[lua] +name: Lua +parsed_name: lua +default_uri: /lua/manual +scraper_name: lua +version: 5.3 +zip_url: +sort: 44 +additional: false +active: true +[marionette] +name: Marionette +parsed_name: marionette +default_uri: /marionette/api +scraper_name: marionette +version: 2.4.3 +zip_url: +sort: 45 +additional: false +active: true +[markdown] +name: MarkDown +parsed_name: markdown +default_uri: /markdown/inline_html +scraper_name: markdown +version: +zip_url: +sort: 46 +additional: false +active: true +[mocha] +name: Mocha +parsed_name: mocha +default_uri: /mocha/index +scraper_name: mocha +version: 2.2.1 +zip_url: +sort: 47 +additional: false +active: true +[modernizr] +name: Modernizr +parsed_name: modernizr +default_uri: /modernizr/index +scraper_name: modernizr +version: 2.8.3 +zip_url: +sort: 48 +additional: false +active: true +[moment] +name: Moment +parsed_name: moment +default_uri: /moment/index +scraper_name: moment +version: 2.10.2 +zip_url: +sort: 49 +additional: false +active: true +[mysql] +name: Mysql +parsed_name: mysql +default_uri: /mysql/index +scraper_name: mysql +version: 5.7 +zip_url: +sort: 51 +additional: false +active: true +[mongoosejs] +name: MongooseJS +parsed_name: mongoosejs +default_uri: /mongoosejs/index +scraper_name: mongoose +version: 4.1.6 +zip_url: +sort: 52 +additional: false +active: true +[nginx] +name: Nginx +parsed_name: nginx +default_uri: /nginx/guide/beginner’s_guide +scraper_name: nginx +version: 1.9.4 +zip_url: +sort: 53 +additional: false +active: true +[node] +name: NodeJS +parsed_name: node +default_uri: /nodejs/assert +scraper_name: node +version: 4.0.0 +zip_url: +sort: 54 +additional: false +active: true +[node_lts] +name: NodeJS LTS +parsed_name: node_lts +default_uri: /nodejs_lts/assert +scraper_name: node_lts +version: 4.2.1 +zip_url: +sort: 55 +additional: false +active: true +[nokogiri] +parsed_name: nokogiri +name: Nokogiri +default_uri: /nokogiri/top_level_namespace +scraper_name: nokogiri2 +version: +zip_url: +sort: 56 +additional: false +active: false +[npm] +name: npm +parsed_name: npm +default_uri: /npm/faq +scraper_name: npm +version: 2.13.0 +zip_url: +sort: 57 +additional: false +active: false +[opentsdb] +name: OpenTSDB +parsed_name: opentsdb +default_uri: /opentsdb/index +scraper_name: opentsdb +version: 2.1.0 +zip_url: +sort: 58 +additional: false +active: true +[perl] +name: Perl +parsed_name: perl +default_uri: /perl/index +scraper_name: perl +version: 5.22 +zip_url: http://perldoc.perl.org/perldoc-html.tar.gz +sort: 59 +additional: false +active: true +[php] +name: PHP +parsed_name: php +default_uri: /php/classes_and_objects +scraper_name: php2 +version: up_to_5.6.2 +zip_url: +sort: 61 +additional: false +active: true +[phpunit] +name: PHPUnit +parsed_name: phpunit +default_uri: /phpunit/phpunit_and_selenium +scraper_name: phpunit +version: 4.8 +zip_url: +sort: 62 +additional: false +active: true +[phalcon] +name: Phalcon +parsed_name: phalcon +default_uri: /phalcon/ +scraper_name: phalcon +version: 2.0.6 +zip_url: +sort: 63 +additional: false +active: true +[phaser] +name: Phaser +parsed_name: phaser +default_uri: /phaser/ +scraper_name: phaser +version: 2.4.1 +zip_url: +sort: 64 +additional: false +active: true +[phoenix] +name: Phoenix +parsed_name: phoenix +default_uri: /phoenix/ +scraper_name: phoenix +version: 1.0.3 +zip_url: +sort: 65 +additional: false +active: false +[postgresql] +name: PostgreSQL +parsed_name: postgresql +default_uri: /postgresql/select +scraper_name: postgresql +version: 9.4 +zip_url: +sort: 66 +additional: false +active: false +[python2] +name: Python2 +parsed_name: python2 +default_uri: /python2/keyword +scraper_name: python2 +version: 2.7.10 +zip_url: https://docs.python.org/2.7/archives/python-2.7.8-docs-html.zip +sort: 67 +additional: false +active: true +[python] +name: Python3 +parsed_name: python3 +default_uri: /python3/abc +scraper_name: python +version: 3.5.0 +zip_url: https://docs.python.org/3/archives/python-3.5.0-docs-html.zip +sort: 68 +additional: false +active: true +[q] +name: q +parsed_name: q +default_uri: /q/index +scraper_name: q +version: 1.4.1 +zip_url: +sort: 69 +additional: false +active: true +[rails] +name: Rails +parsed_name: rails +default_uri: /rails/rails +scraper_name: rails +version: 4.2.0 +zip_url: +sort: 70 +additional: false +active: true +[react] +name: React +parsed_name: react +default_uri: /react/thinking_in_react +scraper_name: react +version: 0.13.3 +zip_url: +sort: 71 +additional: false +active: true +[react_native] +name: React native +parsed_name: react_native +default_uri: /react/ +scraper_name: react_native +version: 0.10.0 +zip_url: +sort: 72 +additional: false +active: true +[react_router] +name: React Router +parsed_name: react_router +default_uri: /react/ +scraper_name: react_router +version: +zip_url: +sort: 73 +additional: false +active: false +[redis] +name: Redis +parsed_name: redis +default_uri: /redis/index +scraper_name: redis +version: up_to_3.0.4 +zip_url: +sort: 74 +additional: false +active: true +[reflux] +name: Reflux +parsed_name: reflux +default_uri: /reflux/index +scraper_name: reflux +version: 0.2.12 +zip_url: +sort: 75 +additional: false +active: true +[relay] +name: Relay +parsed_name: relay +default_uri: /relay/index +scraper_name: relay +version: 0.5 +zip_url: +sort: 76 +additional: false +active: true +[requirejs] +name: RequireJS +parsed_name: requirejs +default_uri: /requirejs/commonjs_notes +scraper_name: requirejs +version: 2.1.18 +zip_url: +sort: 77 +additional: false +active: true +[rethinkdb] +name: RethinkDB +parsed_name: rethinkdb +default_uri: /rethinkdb/index_list +scraper_name: rethinkdb +version: 2.1.2 +zip_url: +sort: 78 +additional: false +active: true +[ruby] +name: Ruby +parsed_name: ruby +default_uri: /ruby/class +scraper_name: ruby +version: 2.2.2 +zip_url: +sort: 79 +additional: false +active: true +[rust] +name: Rust +parsed_name: rust +default_uri: rust/2.2._hello._world! +scraper_name: rust +version: 1.2.0 +zip_url: +sort: 80 +additional: false +active: true +[sass] +name: Sass +parsed_name: sass +default_uri: /sass/list_functions +scraper_name: sass +version: 3.4.7 +zip_url: +sort: 81 +additional: false +active: true +[scala] +name: Scala +parsed_name: scala +default_uri: /scala/index +scraper_name: scala +version: 2.11 +zip_url: +sort: 82 +additional: false +active: true +[sinon] +name: SinonJS +parsed_name: sinonjs +default_uri: /sinonjs/spiapi +scraper_name: sinon +version: 1.16.1 +zip_url: +sort: 83 +additional: false +active: true +[socketio] +name: SocketIO +parsed_name: socketio +default_uri: /socketio/client_api +scraper_name: socketio +version: 1.3.6 +zip_url: +sort: 84 +additional: false +active: true +[sqlite] +name: SQLite +parsed_name: sqlite +default_uri: /sqlite/index +scraper_name: sqlite +version: 3.9.1 +zip_url: https://www.sqlite.org/2015/sqlite-doc-3090100.zip +sort: 85 +additional: false +active: true +[svg] +name: SVG +parsed_name: svg +default_uri: /svg/index +scraper_name: svg +version: +zip_url: +sort: 50 +additional: false +active: true +[swift] +name: Swift +parsed_name: swift +default_uri: /swift/index +scraper_name: swift +version: 12.4 +zip_url: +sort: 86 +additional: false +active: false +[swiftreference] +name: Swift +parsed_name: swift +default_uri: /swift/reference/index +scraper_name: swiftreference +version: 12.4 +zip_url: +sort: 87 +additional: true +active: true +[symfony] +name: Symfony +parsed_name: symfony +default_uri: /symfony/component/console/command/listcommand +scraper_name: symfony +version: 2.7 +zip_url: +sort: 88 +additional: false +active: true +[twig] +name: Twig +parsed_name: twig +default_uri: /twig/doc-index +scraper_name: twig +version: 5.7 +zip_url: +sort: 89 +additional: false +active: true +[underscore] +name: UnderscoreJS +parsed_name: underscorejs +default_uri: /underscorejs/all +scraper_name: underscore +version: 1.8.3 +zip_url: +sort: 90 +additional: false +active: true +[vue] +name: Vue +parsed_name: vue +default_uri: /vue/getting_started +scraper_name: vue +version: 0.12.9 +zip_url: +sort: 91 +additional: false +active: true +[vagrant] +name: Vagrant +parsed_name: vagrant +default_uri: /vagrant/index +scraper_name: vagrant +version: 1.7.4 +zip_url: +sort: 92 +additional: false +active: true +[webpack] +name: Webpack +parsed_name: webpack +default_uri: /webpack/installation +scraper_name: webpack +version: 1.12 +zip_url: +sort: 93 +additional: false +active: true +[xpath] +name: XPath +parsed_name: xpath +default_uri: /xpath/index +scraper_name: xpath +version: +zip_url: +sort: 60 +additional: false +active: true +[yii] +name: Yii +parsed_name: yii +default_uri: /yii/quick_start +scraper_name: yii +version: 2.0.2 +zip_url: +sort: 94 +additional: false +active: true +[yii1] +name: Yii1 +parsed_name: Yii1 +default_uri: /yii1 +scraper_name: yii1 +version: 1.1.16 +zip_url: +sort: 95 +additional: false +active: true diff --git a/lib/docs/db_storage/importer.py b/lib/docs/db_storage/importer.py new file mode 100755 index 0000000000..3d952d123d --- /dev/null +++ b/lib/docs/db_storage/importer.py @@ -0,0 +1,279 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +import json +import psycopg2 +import sys +import ConfigParser +import pprint +import os.path +import re +import time +import math +import logging + +from lxml import html + +class DocImporter(): + connection_string = '' + content_path = '' + debugMode = False + linkerrors = [] + additional = False + docset = '' + filename = '' + version = '' + docset_name = '' + docset_parsed_name = '' + index_path = '' + default_uri = '' + total_entries = 0 + links = {} + + #load config file + def __init__(self, _docset, _mode, _connectionstring): + self.config = ConfigParser.ConfigParser() + self.config.read('importer.cfg') + if(_connectionstring==''): + User = self.config.get('Connection', 'User', 0) + Password = self.config.get('Connection', 'Password', 0) + Host = self.config.get('Connection', 'Host', 0) + DBname = self.config.get('Connection', 'DBname', 0) + self.connection_string = "host='"+ Host + "' dbname='" + DBname + "' user='" + User + "' password='" + Password + "'" + else: + self.connection_string = _connectionstring + self.debugMode = bool(self.config.get('Config', 'debugMode', 0)) + self.content_path = self.config.get('Path', 'base_path', 0) + sections = self.config.sections() + if _mode == 'continue': + sections = sections[sections.index(_docset):] + elif _mode == 'single': + sections = [_docset] + conn = psycopg2.connect(self.connection_string) + self.processDocsets(sections) + + + def processDocsets(self, docsetlist): + for sect in docsetlist: + if sect not in ['Connection', 'Config', 'Path']: + if self.config.get(sect, 'active', 0) == 'true': + try: + self.docset = sect + self.docset_name = self.config.get(sect, 'name', 0) + self.docset_parsed_name = self.config.get(sect, 'parsed_name', 0) + self.default_uri = self.config.get(sect, 'default_uri', 0) + self.additional = self.config.get(sect, 'additional', 0) + self.version = self.config.get(sect, 'version', 0) + if self.version != '': + self.index_path = self.content_path + sect + '~' + self.version + '/index.json' + else: + self.index_path = self.content_path + sect + '/index.json' + print "\rImporting " + sect + "..." + self.importToDB() + except: + lastimportfile = open('lastimport.log', 'w') + lastimportfile.truncate() + lastimportfile.write(sect) + lastimportfile.close(); + + + + def processContent(self, content): + tree = html.fromstring(content) + links = tree.xpath('//a[@href]') + for alink in links: + anchor = '' + keymatch = '' + if alink.get('href')!=None: + match = alink.get('href') + keymatch = match.lower().replace('../', '').replace('%24', '$') + else: + match = '#' + keymatch = '#' + if not keymatch.startswith('http://') and not keymatch.startswith('https://') and not keymatch.startswith('ftp://') and not keymatch.startswith('irc://') and not keymatch.startswith('news://') and not keymatch.startswith('git://') and not keymatch.startswith('mailto:') and keymatch!= '': + if match.find('#')!=-1 and keymatch not in self.links.keys(): + anchor = keymatch[keymatch.find('#'):] + keymatch = keymatch[:keymatch.find('#')] + if keymatch in self.links.keys(): + #print '"' + keymatch + '" - "' + match + '" : "' + self.links[keymatch] + '"' + content = content.replace('"' + match + '"', '"' + self.links[keymatch] + anchor + '"',1) + if keymatch not in self.links and anchor == '' and alink.get('class') != 'broken': + hour = time.strftime("%d/%m/%Y %H:%M:%S") + self.linkerrors.append('- "' + keymatch + '" in ' + self.filename) + return content + + def importToDB(self): + conn = psycopg2.connect(self.connection_string) + try: + json_data = self.processJSON(self.index_path) + self.links = self.createLinkCollection(json_data) + self.initTable(conn) + previous_uri = '' + procesed_entries = {} + total = len(json_data) + i = 1 + for entry in json_data: + _name = entry['name'] + loading_value = int(math.ceil(float(50)/total*i)) + gap_value = 50 - loading_value + loading_bar = ("#" * loading_value) + (" " * gap_value) + sys.stdout.write('\r[' + loading_bar + '](' + str(i)+ '/' + str(total) + ')') + sys.stdout.flush() + i+=1 + if entry['path'].find('#')!= -1: + entry['path'] = entry['path'].split('#')[0] + filename = self.getFileName(self.content_path,self.docset, entry['path']) + #Avoid Process the same Page twice + if entry['path'] in procesed_entries.keys(): + _content = procesed_entries[entry['path']] + else: + self.filename = filename + _content = self.processContent(self.getContent(filename)) + procesed_entries[entry['path']] = _content + + if entry['parent_uri'] == 'null': + _parent_uri = None + else: + _parent_uri = entry['parent_uri'] + _type = entry['type'] + _docset = entry['docset'] + _uri = entry['parsed_uri'] + _anchor = entry['anchor'] + _source_url = entry['source_url'] + if previous_uri != _uri: + self.insertRow(conn, _name, _content, _parent_uri, _type, _docset, _uri, _anchor, _source_url) + previous_uri = _uri + if self.additional == 'false': + self.emptyTable(conn,self.docset_name) + self.moveToData(conn) + self.updateDocsets(conn,self.docset_name, self.default_uri, self.docset_parsed_name) + self.commit(conn) + except Exception, e: + hour = time.strftime("%d/%m/%Y %H:%M:%S") + self.ferrors = open('import_errors.log', 'a') + self.ferrors.write('\n\n\n######################################## ' + self.docset_name + ' ########################################\n\n\n') + self.rollback(conn) + self.initTable(conn) + self.linkerrors = [] + self.ferrors.write('- ' + hour + ' error in ' + self.docset_name + ': %s\n' % e) + self.ferrors.close() + print '========= ERROR =============' + print e + if len(self.linkerrors)>0: + brokenlinksfile = open('brokenlinks_' + self.docset_name +'.log', 'a') + hour = time.strftime("%d/%m/%Y %H:%M:%S") + brokenlinksfile.write('\n\n\n######################################## ' + hour + ' - ' + self.docset_name + ' ########################################\n\n\n') + for error in self.linkerrors: + brokenlinksfile.write(error + '\n') + self.linkerrors = [] + brokenlinksfile.close() + self.finish(conn) + + def createLinkCollection(self, entries): + links = {} + for entry in entries: + if entry['path'].lower() not in links.keys() or entry['anchor']=='': + links[entry['path'].lower()] = entry['parsed_uri'] + if entry['anchor']!= '': + links[entry['path'][entry['path'].find('/'):].lower()] = entry['parsed_uri'] + links[(entry['path'] + '#' + entry['anchor']).lower()] = entry['parsed_uri'] + links['#' + entry['anchor'].lower()] = entry['parsed_uri'] + # EXCEPTION FOR C++ this because the limitations of the documentation into the zip + if entry['docset'].lower() == 'cpp' or entry['docset'].lower() == 'c': + links[entry['path'][entry['path'].find('/')+1:].lower()] = entry['parsed_uri'] + links[entry['path'].replace('fs/', '').replace('io/', '').replace('experimental/', '')] = entry['parsed_uri'] + links[entry['path'].split('/')[-1]] = entry['parsed_uri'] + return links + + def getFileName(self, content_path, docset, path): + if not path.endswith('.html'): + path += '.html' + if self.version != '': + filename = content_path + docset + '~' + self.version + '/' + path + else: + filename = content_path + docset + '/' + path + return filename + + def getContent(self, filename): + with open(filename, 'r') as content_file: + content = content_file.read().decode('utf-8') + return content + + def processJSON(self,file_path): + if os.path.isfile(file_path): + with open(file_path) as json_file: + json_data = json.load(json_file) + self.total_entries = len(json_data['entries']) + return json_data['entries'] + else: + raise Exception("'index.json not found in docset " + self.docset_name) + + def connect(): + return psycopg2.connect(self.connection_string) + + def commit(self, conn): + conn.commit() + print '\n' + self.docset_name + ' imports ' + str(self.total_entries) + '\n' + + def rollback(self, conn): + conn.rollback() + + def insertRow(self, conn, _name, _content, _parent, _type, _docset, _uri, _anchor, _source_url): + pgcursor = conn.cursor() + sqlinsertitem = "INSERT INTO temp_refs (reference, source_url, parent, type, docset, uri, content_anchor) VALUES (%s, %s, %s, %s, %s, %s, %s);" + pgcursor.execute(sqlinsertitem,[_name, + _source_url, + _parent, + _type, + _docset, + _uri, + _anchor]) + + self.removedContent(conn, _source_url) + self.insertContent(conn, _content, _source_url) + + def insertContent(self, conn, _content, _source_url): + pgcursor = conn.cursor() + sqlinsertcontent = "INSERT INTO refs_content (source_url, content) VALUES(%s, %s);" + + pgcursor.execute(sqlinsertcontent,[_source_url, + _content]) + + def moveToData(self, conn): + sqlmovedata = 'INSERT INTO refs (reference, source_url, uri, content_anchor, parent_uri, type, docset) SELECT reference, source_url,uri,content_anchor,parent,type, docset FROM temp_refs;' + pgcursor = conn.cursor() + pgcursor.execute(sqlmovedata) + + + def emptyTable(self, conn, docset): + sqlemptytables = 'DELETE FROM refs WHERE docset=%s;' + pgcursor = conn.cursor() + pgcursor.execute(sqlemptytables, [docset]) + conn.commit() + + def removedContent(self, conn, _source_url): + sqlemptycontent = 'DELETE FROM refs_content WHERE source_url=%s;' + pgcursor = conn.cursor() + pgcursor.execute(sqlemptycontent, [_source_url]) + conn.commit() + + def initTable(self, conn): + sqlinittable = 'TRUNCATE TABLE temp_refs;' + pgcursor = conn.cursor() + pgcursor.execute(sqlinittable) + conn.commit() + + def updateDocsets(self, conn, docset, default_uri, parsed_name): + sqldocsetselect = "SELECT docset FROM docsets WHERE docset = %s;" + sqldocsetinsert = "INSERT INTO docsets (docset, default_uri, pub_date, update_date, active, parsed_name) VALUES (%s,%s,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,%s,%s);" + sqldocsetupdate = "UPDATE docsets SET default_uri = %s, update_date = CURRENT_TIMESTAMP, parsed_name = %s WHERE docset = %s" + pgcursor = conn.cursor() + pgcursor.execute(sqldocsetselect, [docset]) + docsetsrow = pgcursor.fetchone() + if docsetsrow: + pgcursor.execute(sqldocsetupdate, [default_uri, docset, parsed_name]) + else: + pgcursor.execute(sqldocsetinsert,[docset, default_uri, True, parsed_name]) + + def finish(self,conn): + self.initTable(conn) + conn.close() diff --git a/lib/docs/db_storage/lastimport.log b/lib/docs/db_storage/lastimport.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/docs/db_storage/scraper.py b/lib/docs/db_storage/scraper.py new file mode 100644 index 0000000000..8dd7345557 --- /dev/null +++ b/lib/docs/db_storage/scraper.py @@ -0,0 +1,85 @@ +from importer import DocImporter +from collections import OrderedDict +import sys +import time +import subprocess, shlex +import ConfigParser +import pprint + +class Scraper(): + + def __init__(self, _docset, config_file): + self.config = ConfigParser.ConfigParser() + config_file = 'importer.cfg' + self.config.read(config_file) + unorderedsections = self.config.sections() + if _docset!='': + sections = [{'name':_docset, 'sort': 0}] + else: + sections = self.sortSections(unorderedsections) + self.downloadZips(sections) + self.scrapDocsets(sections) + + def sortSections(self, sectlist): + newlist = [] + for sect in sectlist: + if sect not in ['Connection', 'Config', 'Path']: + newitem ={ + 'name' : sect, + 'sort': int(self.config.get(sect, 'sort')) + } + newlist.append(newitem) + orderedsections = sorted(newlist, key=lambda key_value: key_value['sort']) + return orderedsections + + + def downloadZips(self, docsetlist): + working_directory = '../../../' + for sect in docsetlist: + if self.config.get(sect['name'], 'zip_url')!= '': + if self.config.get(sect['name'], 'active') == 'true': + scraper_name = self.config.get(sect['name'], 'scraper_name') + zip_url = self.config.get(sect['name'], 'zip_url') + if zip_url.endswith('.tar.gz'): + extension = 'tar.gz' + else: + extension = zip_url.split('.')[-1] + try: + sys.stdout.flush() + print "Downloading " + zip_url + "..." + subprocess.check_call("wget " + zip_url + " -O ./file_scraper_docs/" + scraper_name + "." + extension, shell=True, cwd=working_directory) + if extension == 'zip': + print "Extracting " + scraper_name + "." + extension + "..." + subprocess.check_call("unzip -o ./file_scraper_docs/" + scraper_name + ".zip -d ./file_scraper_docs/"+ scraper_name + "/", shell=True, cwd=working_directory) + else: + print "Extracting " + scraper_name + "." + extension + "..." + subprocess.check_call("mkdir -p ./file_scraper_docs/" + scraper_name + " && tar -zxvf ./file_scraper_docs/" + scraper_name + ".tar.gz -C ./file_scraper_docs/" + scraper_name, shell=True, cwd=working_directory) + print 'done' + except subprocess.CalledProcessError, e: + hour = time.strftime("%d/%m/%Y %H:%M:%S") + scraper_errors_file = open('scraper_errors.log', 'a') + message = "error downloading " + zip_url + " at" + hour + "\n\n" + str(e) + "###########################################\n\n\n" + scraper_errors_file.write(message) + scraper_errors_file.close() + + def scrapDocsets(self, docsetlist): + working_directory = '../../../' + for sect in docsetlist: + if self.config.get(sect['name'], 'active') == 'true': + scraper_name = self.config.get(sect['name'], 'scraper_name') + print "Processing " + scraper_name + "..." + try: + subprocess.check_call("thor docs generate " + scraper_name + " --force",shell=True, cwd=working_directory) + print 'done' + except subprocess.CalledProcessError, e: + hour = time.strftime("%d/%m/%Y %H:%M:%S") + scraper_errors_file = open('scraper_errors.log', 'a') + message = "error scrapping " + scraper_name + " at" + hour + "\n\n" + str(e) + "###########################################\n\n\n" + scraper_errors_file.write(message) + scraper_errors_file.close() + except Exception, e: + hour = time.strftime("%d/%m/%Y %H:%M:%S") + scraper_errors_file = open('scraper_errors.log', 'a') + message = "error scrapping " + scraper_name + " at" + hour + "\n\n" + str(e) + "###########################################\n\n\n" + scraper_errors_file.write(message) + scraper_errors_file.close() diff --git a/lib/docs/db_storage/tests/data/test_backbone.html b/lib/docs/db_storage/tests/data/test_backbone.html new file mode 100644 index 0000000000..35abd84aa1 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_backbone.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_backbone.json b/lib/docs/db_storage/tests/data/test_backbone.json new file mode 100644 index 0000000000..83a29f176d --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_backbone.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"Backbone.on","path":"index","type":"event","parsed_uri":"/backbonejs/backbone.on","anchor":"Events-on","parent_uri":"null","docset":"BackboneJS"}, +{"name":"Backbone.listenTo","path":"index","type":"event","parsed_uri":"/backbonejs/backbone.listento","anchor":"Events-listenTo","parent_uri":"null","docset":"BackboneJS"}, +{"name":"model.validate","path":"index","type":"function","parsed_uri":"/backbonejs/model.validate","anchor":"Model-validate","parent_uri":"null","docset":"BackboneJS"}, +{"name":"model.destroy","path":"index","type":"function","parsed_uri":"/backbonejs/model.destroy","anchor":"Model-destroy","parent_uri":"null","docset":"BackboneJS"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_bower.html b/lib/docs/db_storage/tests/data/test_bower.html new file mode 100644 index 0000000000..0ea46e844a --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_bower.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_bower.json b/lib/docs/db_storage/tests/data/test_bower.json new file mode 100644 index 0000000000..134b389bea --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_bower.json @@ -0,0 +1,8 @@ +{"entries":[ +{"name":"analytics","path":"config","type":"configuration","parsed_uri":"/bower/analytics","anchor":"analytics","parent_uri":"null","docset":"Bower"}, +{"name":"cwd","path":"config","type":"configuration","parsed_uri":"/bower/cwd","anchor":"cwd","parent_uri":"null","docset":"Bower"}, +{"name":"https-proxy","path":"config","type":"configuration","parsed_uri":"/bower/https-proxy","anchor":"https-proxy","parent_uri":"null","docset":"Bower"}, +{"name":"color","path":"config","type":"configuration","parsed_uri":"/bower/color","anchor":"color","parent_uri":"null","docset":"Bower"}, +{"name":"Register","path":"creating-packages","type":"guide","parsed_uri":"/bower/register","anchor":"register","parent_uri":"null","docset":"Bower"}, +{"name":"init","path":"api","type":"api","parsed_uri":"/bower/init","anchor":"init","parent_uri":"null","docset":"Bower"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_c.html b/lib/docs/db_storage/tests/data/test_c.html new file mode 100644 index 0000000000..9e19a8364a --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_c.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_c.json b/lib/docs/db_storage/tests/data/test_c.json new file mode 100644 index 0000000000..0c141b5394 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_c.json @@ -0,0 +1,7 @@ +{"entries":[ +{"name":"Variadic functions","path":"variadic","type":"others","parsed_uri":"/c/variadic_functions","anchor":"","parent_uri":"null","docset":"C"}, +{"name":"Error handling","path":"error","type":"others","parsed_uri":"/c/error_handling","anchor":"","parent_uri":"null","docset":"C"}, +{"name":"Numeric limits","path":"types/limits","type":"type","parsed_uri":"/c/type_support/numeric_limits","anchor":"","parent_uri":"/c/type_support","docset":"C"}, +{"name":"Common mathematical functions","path":"numeric/math","type":"others","parsed_uri":"/c/numerics/common_mathematical_functions","anchor":"","parent_uri":"/c/numerics","docset":"C"}, +{"name":"Program support utilities","path":"program","type":"others","parsed_uri":"/c/program_support_utilities","anchor":"","parent_uri":"null","docset":"C"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_chai.html b/lib/docs/db_storage/tests/data/test_chai.html new file mode 100644 index 0000000000..c4f3066712 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_chai.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_chai.json b/lib/docs/db_storage/tests/data/test_chai.json new file mode 100644 index 0000000000..7fc8895956 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_chai.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"Core Plugin Concepts","path":"guide/plugins/index","type":"plugin","parsed_uri":"/chai/core_plugin_concepts","anchor":"","parent_uri":"null","docset":"Chai"}, +{"name":"addProperty","path":"api/plugins/index","type":"plugin","parsed_uri":"/chai/addproperty","anchor":"addProperty-section","parent_uri":"null","docset":"Chai"}, +{"name":"addMethod","path":"api/plugins/index","type":"plugin","parsed_uri":"/chai/addmethod","anchor":"addMethod-section","parent_uri":"null","docset":"Chai"}, +{"name":"Building a Helper","path":"guide/helpers/index","type":"others","parsed_uri":"/chai/building_a_helper","anchor":"","parent_uri":"null","docset":"Chai"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_cordova.html b/lib/docs/db_storage/tests/data/test_cordova.html new file mode 100644 index 0000000000..f7df186c25 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_cordova.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_cordova.json b/lib/docs/db_storage/tests/data/test_cordova.json new file mode 100644 index 0000000000..4a98c0b061 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_cordova.json @@ -0,0 +1,7 @@ +{"entries":[ +{"name":"Amazon Fire OS Configuration","path":"guide_platforms_amazonfireos_config.md","type":"guide","parsed_uri":"/apache-cordova/amazon_fire_os_configuration","anchor":"","parent_uri":"null","docset":"Apache Cordova"}, +{"name":"Amazon Fire OS WebViews","path":"guide_platforms_amazonfireos_webview.md","type":"guide","parsed_uri":"/apache-cordova/amazon_fire_os_webviews","anchor":"","parent_uri":"null","docset":"Apache Cordova"}, +{"name":"Amazon Fire OS Plugins","path":"guide_platforms_amazonfireos_plugin.md","type":"guide","parsed_uri":"/apache-cordova/amazon_fire_os_plugins","anchor":"","parent_uri":"null","docset":"Apache Cordova"}, +{"name":"Overview","path":"guide_overview_index.md","type":"guide","parsed_uri":"/apache-cordova/overview","anchor":"","parent_uri":"null","docset":"Apache Cordova"}, +{"name":"The Command-Line Interface","path":"guide_cli_index.md","type":"guide","parsed_uri":"/apache-cordova/the_command-line_interface","anchor":"","parent_uri":"null","docset":"Apache Cordova"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_cpp.html b/lib/docs/db_storage/tests/data/test_cpp.html new file mode 100644 index 0000000000..b64d2b409a --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_cpp.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_cpp.json b/lib/docs/db_storage/tests/data/test_cpp.json new file mode 100644 index 0000000000..4c813b918f --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_cpp.json @@ -0,0 +1,9 @@ +{"entries":[ +{"name":"path","path":"experimental/fs/path","type":"others","parsed_uri":"/cpp/filesystem_library/path","anchor":"","parent_uri":"/cpp/filesystem_library","docset":"Cpp"}, +{"name":"filesystem_error","path":"experimental/fs/filesystem_error","type":"others","parsed_uri":"/cpp/filesystem_library/filesystem_error","anchor":"","parent_uri":"/cpp/filesystem_library","docset":"Cpp"}, +{"name":"filesystem_error::filesystem_error","path":"experimental/fs/filesystem_error/filesystem_error","type":"others","parsed_uri":"/cpp/filesystem_library/filesystem_error/filesystem_error--filesystem_error","anchor":"","parent_uri":"/cpp/filesystem_library/filesystem_error","docset":"Cpp"}, +{"name":"Algorithms library","path":"algorithm","type":"Algorithm","parsed_uri":"/cpp/algorithms_library","anchor":"","parent_uri":"null","docset":"Cpp"}, +{"name":"FP_INFINITE","path":"numeric/math/fp_categories","type":"others","parsed_uri":"/cpp/numerics_library/common_mathematical_functions/fp_normal._fp_subnormal._fp_zero._fp_infinite._fp_nan/fp_infinite","anchor":"","parent_uri":"/cpp/numerics_library/common_mathematical_functions/fp_normal._fp_subnormal._fp_zero._fp_infinite._fp_nan","docset":"Cpp"}, +{"name":"ctime","path":"header/ctime","type":"Standard library header files","parsed_uri":"/cpp/standard_library_header_files/ctime","anchor":"","parent_uri":"/cpp/standard_library_header_files","docset":"Cpp"}, +{"name":"chrono","path":"header/chrono","type":"Standard library header files","parsed_uri":"/cpp/standard_library_header_files/chrono","anchor":"","parent_uri":"/cpp/standard_library_header_files","docset":"Cpp"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_css.html b/lib/docs/db_storage/tests/data/test_css.html new file mode 100644 index 0000000000..99f0671f64 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_css.html @@ -0,0 +1 @@ +height, box-sizing, min-width, max-widthpercentage \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_css.json b/lib/docs/db_storage/tests/data/test_css.json new file mode 100644 index 0000000000..266ec5d640 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_css.json @@ -0,0 +1,7 @@ +{"entries":[ +{"name":"height","path":"height","type":"property","parsed_uri":"/css/height","anchor":"","parent_uri":"null","docset":"CSS"}, +{"name":"box-sizing","path":"box-sizing","type":"property","parsed_uri":"/css/box-sizing","anchor":"","parent_uri":"null","docset":"CSS"}, +{"name":"min-width","path":"min-width","type":"property","parsed_uri":"/css/min-width","anchor":"","parent_uri":"null","docset":"CSS"}, +{"name":"max-width","path":"max-width","type":"property","parsed_uri":"/css/max-width","anchor":"","parent_uri":"null","docset":"CSS"}, +{"name":"","path":"percentage","type":"type","parsed_uri":"/css/_percentage_","anchor":"","parent_uri":"null","docset":"CSS"} +]} diff --git a/lib/docs/db_storage/tests/data/test_d3.html b/lib/docs/db_storage/tests/data/test_d3.html new file mode 100644 index 0000000000..ceeab085b1 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_d3.html @@ -0,0 +1 @@ +
attributes, styles, properties, HTML and text
\ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_d3.json b/lib/docs/db_storage/tests/data/test_d3.json new file mode 100644 index 0000000000..3fb97d94d1 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_d3.json @@ -0,0 +1,7 @@ +{"entries":[ +{"name":"selection.attr","path":"selections","type":"others","parsed_uri":"/d3/selection.attr","anchor":"attr","parent_uri":"null","docset":"D3"}, +{"name":"selection.style","path":"selections","type":"others","parsed_uri":"/d3/selection.style","anchor":"style","parent_uri":"null","docset":"D3"}, +{"name":"selection.property","path":"selections","type":"others","parsed_uri":"/d3/selection.property","anchor":"property","parent_uri":"null","docset":"D3"}, +{"name":"selection.html","path":"selections","type":"others","parsed_uri":"/d3/selection.html","anchor":"html","parent_uri":"null","docset":"D3"}, +{"name":"selection.text","path":"selections","type":"others","parsed_uri":"/d3/selection.text","anchor":"text","parent_uri":"null","docset":"D3"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_dom.html b/lib/docs/db_storage/tests/data/test_dom.html new file mode 100644 index 0000000000..3bd2d47554 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_dom.html @@ -0,0 +1 @@ +EventTargetdocument.loadDocument.compatModeDocument.domConfigDocument.implementation \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_dom.json b/lib/docs/db_storage/tests/data/test_dom.json new file mode 100644 index 0000000000..11b0e37c1e --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_dom.json @@ -0,0 +1,7 @@ +{"entries":[ +{"name":"EventTarget","path":"eventtarget","type":"object","parsed_uri":"/dom/eventtarget","anchor":"","parent_uri":"null","docset":"DOM"}, +{"name":"Document.loadOverlay","path":"document/loadoverlay","type":"others","parsed_uri":"/dom/document/loadoverlay","anchor":"","parent_uri":"/dom/document","docset":"DOM"}, +{"name":"Document.compatMode","path":"document/compatmode","type":"others","parsed_uri":"/dom/document/compatmode","anchor":"","parent_uri":"/dom/document","docset":"DOM"}, +{"name":"Document.compatMode","path":"document/compatmode","type":"others","parsed_uri":"/dom/document/compatmode","anchor":"","parent_uri":"/dom/document","docset":"DOM"}, +{"name":"Document.implementation","path":"document/implementation","type":"object","parsed_uri":"/dom/document/implementation","anchor":"","parent_uri":"/dom/document","docset":"DOM"} +]} diff --git a/lib/docs/db_storage/tests/data/test_dom_events.html b/lib/docs/db_storage/tests/data/test_dom_events.html new file mode 100644 index 0000000000..018f095c9b --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_dom_events.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_dom_events.json b/lib/docs/db_storage/tests/data/test_dom_events.json new file mode 100644 index 0000000000..c8b8c757f1 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_dom_events.json @@ -0,0 +1,8 @@ +{"entries":[ +{"name":"loadstart","path":"loadstart","type":"event","parsed_uri":"/dom_events/loadstart","anchor":"","parent_uri":"null","docset":"DOM Events"}, +{"name":"error","path":"error_(progressevent)","type":"event","parsed_uri":"/dom_events/error_(progressevent)","anchor":"","parent_uri":"null","docset":"DOM Events"}, +{"name":"progress","path":"progress","type":"event","parsed_uri":"/dom_events/progress","anchor":"","parent_uri":"null","docset":"DOM Events"}, +{"name":"abort","path":"abort_(progressevent)","type":"event","parsed_uri":"/dom_events/abort_(progressevent)","anchor":"","parent_uri":"null","docset":"DOM Events"}, +{"name":"load","path":"load","type":"event","parsed_uri":"/dom_events/load","anchor":"","parent_uri":"null","docset":"DOM Events"}, +{"name":"loadend","path":"loadend","type":"event","parsed_uri":"/dom_events/loadend","anchor":"","parent_uri":"null","docset":"DOM Events"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_ember.html b/lib/docs/db_storage/tests/data/test_ember.html new file mode 100644 index 0000000000..4abd4d47d2 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_ember.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_ember.json b/lib/docs/db_storage/tests/data/test_ember.json new file mode 100644 index 0000000000..04fecf9bf0 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_ember.json @@ -0,0 +1,5 @@ +{"entries":[ +{"name":"ember-views","path":"modules/ember-views","type":"modules","parsed_uri":"/ember/modules/ember-views","anchor":"","parent_uri":"/ember/modules","docset":"EmberJS"}, +{"name":"View","path":"classes/ember.view","type":"view","parsed_uri":"/ember/classes/view","anchor":"","parent_uri":"/ember/classes","docset":"EmberJS"}, +{"name":"View.views","path":"classes/ember.view","type":"view","parsed_uri":"/ember/classes/view.views","anchor":"property_views","parent_uri":"/ember/classes/view","docset":"EmberJS"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_express.html b/lib/docs/db_storage/tests/data/test_express.html new file mode 100644 index 0000000000..b43690ac02 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_express.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_express.json b/lib/docs/db_storage/tests/data/test_express.json new file mode 100644 index 0000000000..9e1e771660 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_express.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"req.baseUrl","path":"index","type":"request","parsed_uri":"/express/req.baseurl","anchor":"req.baseUrl","parent_uri":"null","docset":"Express"}, +{"name":"app.route()","path":"index","type":"application","parsed_uri":"/express/app.route","anchor":"app.route","parent_uri":"null","docset":"Express"}, +{"name":"app.METHOD()","path":"index","type":"application","parsed_uri":"/express/app.method","anchor":"app.METHOD","parent_uri":"null","docset":"Express"}, +{"name":"app.render()","path":"index","type":"application","parsed_uri":"/express/app.render","anchor":"app.render","parent_uri":"null","docset":"Express"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_git.html b/lib/docs/db_storage/tests/data/test_git.html new file mode 100644 index 0000000000..bf89d8e3b8 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_git.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_git.json b/lib/docs/db_storage/tests/data/test_git.json new file mode 100644 index 0000000000..7634ae0a56 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_git.json @@ -0,0 +1,8 @@ +{"entries":[ +{"name":"gitattributes","path":"gitattributes","type":"function","parsed_uri":"/git/gitattributes","anchor":"","parent_uri":"null","docset":"Git"}, +{"name":"git annotate","path":"git-annotate","type":"function","parsed_uri":"/git/git-annotate","anchor":"","parent_uri":"null","docset":"Git"}, +{"name":"git upload-archive","path":"git-upload-archive","type":"function","parsed_uri":"/git/git-upload-archive","anchor":"","parent_uri":"null","docset":"Git"}, +{"name":"git blame","path":"git-blame","type":"lists","parsed_uri":"/git/git-blame","anchor":"","parent_uri":"null","docset":"Git"}, +{"name":"git-http-backend","path":"git-http-backend","type":"function","parsed_uri":"/git/git-http-backend","anchor":"","parent_uri":"null","docset":"Git"}, +{"name":"git log","path":"git-log","type":"lists","parsed_uri":"/git/git-log","anchor":"","parent_uri":"null","docset":"Git"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_go.html b/lib/docs/db_storage/tests/data/test_go.html new file mode 100644 index 0000000000..40e9b1e4a6 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_go.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_go.json b/lib/docs/db_storage/tests/data/test_go.json new file mode 100644 index 0000000000..d2881cb331 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_go.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"Sub-repositories","path":"index","type":"others","parsed_uri":"/go/go_programming_language/sub-repositories","anchor":"subrepo","parent_uri":"null","docset":"Go"}, +{"name":"Writer (zip)","path":"archive/zip/index","type":"function","parsed_uri":"/go/archive/zip/writer_zip","anchor":"Writer","parent_uri":"/go/archive/zip","docset":"Go"}, +{"name":"lzw","path":"compress/lzw/index","type":"function","parsed_uri":"/go/compress/lzw","anchor":"","parent_uri":"/go/compress","docset":"Go"}, +{"name":"Map() (bytes)","path":"bytes/index","type":"type","parsed_uri":"/go/bytes/map_bytes","anchor":"Map","parent_uri":"/go/bytes","docset":"Go"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_grunt.html b/lib/docs/db_storage/tests/data/test_grunt.html new file mode 100644 index 0000000000..ea6f2da7c4 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_grunt.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_grunt.json b/lib/docs/db_storage/tests/data/test_grunt.json new file mode 100644 index 0000000000..f8af70d0ed --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_grunt.json @@ -0,0 +1,4 @@ +{"entries":[ +{"name":"Using the CLI","path":"using-the-cli","type":"others","parsed_uri":"/grunt/using-the-cli","anchor":"","parent_uri":"null","docset":"Grunt"}, +{"name":"grunt","path":"api/grunt","type":"others","parsed_uri":"/grunt/api/grunt","anchor":"","parent_uri":null,"docset":"Grunt"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_haskell.html b/lib/docs/db_storage/tests/data/test_haskell.html new file mode 100644 index 0000000000..9622e3604f --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_haskell.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_haskell.json b/lib/docs/db_storage/tests/data/test_haskell.json new file mode 100644 index 0000000000..8c938d251d --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_haskell.json @@ -0,0 +1,7 @@ +{"entries":[ +{"name":"mapWithKey :: (k -> a -> b) -> Map k a -> Map k b","path":"containers-0.5.5.1/data-map-strict","type":"data","parsed_uri":"/haskell/data-map-strict/mapwithkey_--_k_->_a_->_b_->_map_k_a_->_map_k_b","anchor":"v:mapWithKey","parent_uri":"/haskell/data-map-strict","docset":"Haskell"}, +{"name":"IntMap a","path":"containers-0.5.5.1/data-intmap-strict","type":"data","parsed_uri":"/haskell/data-intmap-strict/intmap_a","anchor":"t:IntMap","parent_uri":"/haskell/data-intmap-strict","docset":"Haskell"}, +{"name":"Key = Int","path":"containers-0.5.5.1/data-intset","type":"data","parsed_uri":"/haskell/data-intset/key_=_int","anchor":"t:Key","parent_uri":"/haskell/data-intset","docset":"Haskell"}, +{"name":"Data.IntMap.Lazy","path":"containers-0.5.5.1/data-intmap-lazy","type":"data","parsed_uri":"/haskell/data-intmap-lazy","anchor":"","parent_uri":"/haskell/data-intmap-lazy","docset":"Haskell"}, +{"name":"foldrWithKey :: (k -> a -> b -> b) -> b -> Map k a -> b","path":"containers-0.5.5.1/data-map-strict","type":"data","parsed_uri":"/haskell/data-map-strict/foldrwithkey_--_k_->_a_->_b_->_b_->_b_->_map_k_a_->_b","anchor":"v:foldrWithKey","parent_uri":"/haskell/data-map-strict","docset":"Haskell"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_html.html b/lib/docs/db_storage/tests/data/test_html.html new file mode 100644 index 0000000000..8caa7bce46 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_html.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_html.json b/lib/docs/db_storage/tests/data/test_html.json new file mode 100644 index 0000000000..4c593f77d1 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_html.json @@ -0,0 +1,10 @@ +{"entries":[ +{"name":"global attributes","path":"global_attributes","type":"standard","parsed_uri":"/html/global_attributes","anchor":"","parent_uri":"null","docset":"HTML"}, +{"name":"meter","path":"element/meter","type":"type","parsed_uri":"/html/element/meter","anchor":"","parent_uri":"/html/element","docset":"HTML"}, +{"name":"basefont","path":"element/basefont","type":"obsolete","parsed_uri":"/html/element/basefont","anchor":"","parent_uri":"/html/element","docset":"HTML"}, +{"name":"base","path":"element/base","type":"standard","parsed_uri":"/html/element/base","anchor":"","parent_uri":"/html/element","docset":"HTML"}, +{"name":"html","path":"element/html","type":"standard","parsed_uri":"/html/element/html","anchor":"","parent_uri":"/html/element","docset":"HTML"}, +{"name":"option","path":"element/option","type":"standard","parsed_uri":"/html/element/option","anchor":"","parent_uri":"/html/element","docset":"HTML"}, +{"name":"link","path":"element/link","type":"standard","parsed_uri":"/html/element/link","anchor":"","parent_uri":"/html/element","docset":"HTML"}, +{"name":"meter","path":"element/meter","type":"type","parsed_uri":"/html/element/meter","anchor":"","parent_uri":"/html/element","docset":"HTML"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_javascript.html b/lib/docs/db_storage/tests/data/test_javascript.html new file mode 100644 index 0000000000..d93f22a12c --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_javascript.html @@ -0,0 +1 @@ +
Block
break
continue
Empty
if...else
switch
throw
try...catch
try...catch
try...catch
arguments
\ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_javascript.json b/lib/docs/db_storage/tests/data/test_javascript.json new file mode 100644 index 0000000000..71157a4f96 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_javascript.json @@ -0,0 +1,13 @@ +{"entries":[ +{"name":"P1","path":"statements/block","type":"method","parsed_uri":"/javascript/statements/block","anchor":"","parent_uri":"/javascript/statements","docset":"JavaScript"}, +{"name":"P2","path":"statements/break","type":"method","parsed_uri":"/javascript/statements/break","anchor":"","parent_uri":"/javascript/statements","docset":"JavaScript"}, +{"name":"P3","path":"statements/continue","type":"method","parsed_uri":"/javascript/statements/continue","anchor":"","parent_uri":"/javascript/statements","docset":"JavaScript"}, +{"name":"P4","path":"statements/empty","type":"method","parsed_uri":"/javascript/statements/empty","anchor":"","parent_uri":"/javascript/statements","docset":"JavaScript"}, +{"name":"P5","path":"statements/if...else","type":"method","parsed_uri":"/javascript/statements/if...else","anchor":"","parent_uri":"/javascript/statements","docset":"JavaScript"}, +{"name":"P6","path":"statements/switch","type":"method","parsed_uri":"/javascript/statements/switch","anchor":"","parent_uri":"/javascript/statements","docset":"JavaScript"}, +{"name":"P7","path":"statements/throw","type":"method","parsed_uri":"/javascript/statements/throw","anchor":"","parent_uri":"/javascript/statements","docset":"JavaScript"}, +{"name":"P8","path":"statements/try...catch","type":"method","parsed_uri":"/javascript/statements/try...catch","anchor":"","parent_uri":"/javascript/statements","docset":"JavaScript"}, +{"name":"P9","path":"operators/yield*","type":"method","parsed_uri":"/javascript/operators/yield+","anchor":"","parent_uri":"/javascript/operators","docset":"JavaScript"}, +{"name":"P10","path":"operators/yield","type":"method","parsed_uri":"/javascript/operators/yield","anchor":"","parent_uri":"/javascript/operators","docset":"JavaScript"}, +{"name":"P11","path":"functions/arguments","type":"method","parsed_uri":"/javascript/functions/arguments","anchor":"","parent_uri":"/javascript/functions","docset":"JavaScript"} +]} diff --git a/lib/docs/db_storage/tests/data/test_jquery.html b/lib/docs/db_storage/tests/data/test_jquery.html new file mode 100644 index 0000000000..1e989f9af8 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_jquery.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_jquery.json b/lib/docs/db_storage/tests/data/test_jquery.json new file mode 100644 index 0000000000..a1ed88d82a --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_jquery.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"jQuery.each()","path":"jquery.each","type":"function","parsed_uri":"/jquery/jquery.each","anchor":"","parent_uri":"null","docset":"JQuery"}, +{"name":"jQuery.merge()","path":"jquery.merge","type":"function","parsed_uri":"/jquery/jquery.merge","anchor":"","parent_uri":"null","docset":"JQuery"}, +{"name":"jQuery.parseHTML()","path":"jquery.parsehtml","type":"function","parsed_uri":"/jquery/jquery.parsehtml","anchor":"","parent_uri":"null","docset":"JQuery"}, +{"name":"jQuery.proxy()","path":"jquery.proxy","type":"function","parsed_uri":"/jquery/jquery.proxy","anchor":"","parent_uri":"null","docset":"JQuery"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_jquerymobile.html b/lib/docs/db_storage/tests/data/test_jquerymobile.html new file mode 100644 index 0000000000..7a3953964c --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_jquerymobile.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_jquerymobile.json b/lib/docs/db_storage/tests/data/test_jquerymobile.json new file mode 100644 index 0000000000..5d88a4d3c6 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_jquerymobile.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"Configuring Defaults","path":"global-config","type":"others","parsed_uri":"/jquerymobile/configuring_defaults","anchor":"","parent_uri":"null","docset":"JQuery Mobile"}, +{"name":"Collapsible","path":"collapsible","type":"function","parsed_uri":"/jquerymobile/collapsible","anchor":"","parent_uri":"null","docset":"JQuery Mobile"}, +{"name":"Collapsibleset","path":"collapsibleset","type":"function","parsed_uri":"/jquerymobile/collapsibleset","anchor":"","parent_uri":"null","docset":"JQuery Mobile"}, +{"name":"Flipswitch","path":"flipswitch","type":"function","parsed_uri":"/jquerymobile/flipswitch","anchor":"","parent_uri":"null","docset":"JQuery Mobile"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_jqueryui.html b/lib/docs/db_storage/tests/data/test_jqueryui.html new file mode 100644 index 0000000000..b072eae6ff --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_jqueryui.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_jqueryui.json b/lib/docs/db_storage/tests/data/test_jqueryui.json new file mode 100644 index 0000000000..4f7c972936 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_jqueryui.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"Widget factory","path":"jquery.widget","type":"function","parsed_uri":"/jqueryui/jquery.widget","anchor":"","parent_uri":"null","docset":"JQuery UI"}, +{"name":"CSS framework","path":"theming/css-framework","type":"others","parsed_uri":"/jqueryui/theming/css-framework","anchor":"","parent_uri":"/jqueryui/theming","docset":"JQuery UI"}, +{"name":"Easings","path":"easings","type":"others","parsed_uri":"/jqueryui/easings","anchor":"","parent_uri":"null","docset":"JQuery UI"}, +{"name":"Icons","path":"theming/icons","type":"others","parsed_uri":"/jqueryui/theming/icons","anchor":"","parent_uri":"/jqueryui/theming","docset":"JQuery UI"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_less.html b/lib/docs/db_storage/tests/data/test_less.html new file mode 100644 index 0000000000..62c245ad8e --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_less.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_less.json b/lib/docs/db_storage/tests/data/test_less.json new file mode 100644 index 0000000000..0e27a46149 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_less.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"data-uri","path":"functions","type":"function","parsed_uri":"/less/data-uri","anchor":"misc-functions-data-uri","parent_uri":"null","docset":"Less"}, +{"name":"default","path":"functions","type":"function","parsed_uri":"/less/default","anchor":"misc-functions-default","parent_uri":"null","docset":"Less"}, +{"name":"desaturate","path":"functions","type":"function","parsed_uri":"/less/desaturate","anchor":"color-operations-desaturate","parent_uri":"null","docset":"Less"}, +{"name":"difference","path":"functions","type":"function","parsed_uri":"/less/difference","anchor":"color-blending-difference","parent_uri":"null","docset":"Less"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_lodash.html b/lib/docs/db_storage/tests/data/test_lodash.html new file mode 100644 index 0000000000..e4a0d38c3a --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_lodash.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_lodash.json b/lib/docs/db_storage/tests/data/test_lodash.json new file mode 100644 index 0000000000..b64ef962ed --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_lodash.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"_.detect()","path":"index","type":"collection","parsed_uri":"/lodash/_.detect","anchor":"find","parent_uri":"null","docset":"Lo-Dash"}, +{"name":"_.matchesProperty()","path":"index","type":"function","parsed_uri":"/lodash/_.matchesproperty","anchor":"matchesProperty","parent_uri":"null","docset":"Lo-Dash"}, +{"name":"_.property()","path":"index","type":"function","parsed_uri":"/lodash/_.property","anchor":"property","parent_uri":"null","docset":"Lo-Dash"}, +{"name":"_.matches()","path":"index","type":"function","parsed_uri":"/lodash/_.matches","anchor":"matches","parent_uri":"null","docset":"Lo-Dash"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_markdown.html b/lib/docs/db_storage/tests/data/test_markdown.html new file mode 100644 index 0000000000..eb9c3a9c69 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_markdown.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_markdown.json b/lib/docs/db_storage/tests/data/test_markdown.json new file mode 100644 index 0000000000..0f678eec5b --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_markdown.json @@ -0,0 +1,12 @@ +{"entries":[ +{"name":"Overview","path":"index","type":"others","parsed_uri":"/markdown/overview","anchor":"overview","parent_uri":"null","docset":"MarkDown"}, +{"name":"Philosophy","path":"index","type":"guide","parsed_uri":"/markdown/philosophy","anchor":"philosophy","parent_uri":"null","docset":"MarkDown"}, +{"name":"Inline HTML","path":"index","type":"guide","parsed_uri":"/markdown/inline_html","anchor":"html","parent_uri":"null","docset":"MarkDown"}, +{"name":"Automatic Escaping for Special Characters","path":"index","type":"guide","parsed_uri":"/markdown/automatic_escaping_for_special_characters","anchor":"autoescape","parent_uri":"null","docset":"MarkDown"}, +{"name":"Paragraphs and Line Breaks","path":"index","type":"element","parsed_uri":"/markdown/paragraphs_and_line_breaks","anchor":"p","parent_uri":"null","docset":"MarkDown"}, +{"name":"Headers","path":"index","type":"element","parsed_uri":"/markdown/headers","anchor":"header","parent_uri":"null","docset":"MarkDown"}, +{"name":"Blockquotes","path":"index","type":"element","parsed_uri":"/markdown/blockquotes","anchor":"blockquote","parent_uri":"null","docset":"MarkDown"}, +{"name":"Lists","path":"index","type":"element","parsed_uri":"/markdown/lists","anchor":"list","parent_uri":"null","docset":"MarkDown"}, +{"name":"Code Blocks","path":"index","type":"element","parsed_uri":"/markdown/code_blocks","anchor":"precode","parent_uri":"null","docset":"MarkDown"}, +{"name":"Horizontal Rules","path":"index","type":"element","parsed_uri":"/markdown/horizontal_rules","anchor":"hr","parent_uri":"null","docset":"MarkDown"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_mongoosejs.html b/lib/docs/db_storage/tests/data/test_mongoosejs.html new file mode 100644 index 0000000000..74685aae2d --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_mongoosejs.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_mongoosejs.json b/lib/docs/db_storage/tests/data/test_mongoosejs.json new file mode 100644 index 0000000000..69946267e3 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_mongoosejs.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"Middleware","path":"middleware","type":"guide","parsed_uri":"/mongoosejs/middleware","anchor":"","parent_uri":"null","docset":"MongooseJS"}, +{"name":"Schema.add()","path":"api","type":"schema","parsed_uri":"/mongoosejs/schema.add","anchor":"schema_Schema-add","parent_uri":"null","docset":"MongooseJS"}, +{"name":"SchemaType()","path":"api","type":"schema","parsed_uri":"/mongoosejs/schematype","anchor":"schematype_SchemaType","parent_uri":"null","docset":"MongooseJS"}, +{"name":"SchemaType.index()","path":"api","type":"schema","parsed_uri":"/mongoosejs/schematype.index","anchor":"schematype_SchemaType-index","parent_uri":"null","docset":"MongooseJS"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_nginx.html b/lib/docs/db_storage/tests/data/test_nginx.html new file mode 100644 index 0000000000..4819079f68 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_nginx.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_nginx.json b/lib/docs/db_storage/tests/data/test_nginx.json new file mode 100644 index 0000000000..0bceebc3d3 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_nginx.json @@ -0,0 +1,9 @@ +{"entries":[ +{"name":"debug_connection","path":"ngx_core_module","type":"core","parsed_uri":"/nginx/debug_connection","anchor":"debug_connection","parent_uri":"/nginx/core","docset":"Nginx"}, +{"name":"autoindex_format","path":"http/ngx_http_autoindex_module","type":"module","parsed_uri":"/nginx/autoindex_format","anchor":"autoindex_format","parent_uri":"/nginx/module","docset":"Nginx"}, +{"name":"server","path":"http/ngx_http_core_module","type":"core","parsed_uri":"/nginx/server","anchor":"server","parent_uri":"/nginx/core","docset":"Nginx"}, +{"name":"error_log","path":"ngx_core_module","type":"core","parsed_uri":"/nginx/error_log","anchor":"error_log","parent_uri":"/nginx/core","docset":"Nginx"}, +{"name":"auth_request_set","path":"http/ngx_http_auth_request_module","type":"module","parsed_uri":"/nginx/auth_request_set","anchor":"auth_request_set","parent_uri":"/nginx/module","docset":"Nginx"}, +{"name":"autoindex","path":"http/ngx_http_autoindex_module","type":"module","parsed_uri":"/nginx/autoindex","anchor":"autoindex","parent_uri":"/nginx/module","docset":"Nginx"}, +{"name":"autoindex_format","path":"http/ngx_http_autoindex_module","type":"module","parsed_uri":"/nginx/autoindex_format","anchor":"autoindex_format","parent_uri":"/nginx/module","docset":"Nginx"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_node.html b/lib/docs/db_storage/tests/data/test_node.html new file mode 100644 index 0000000000..ad5e2500cb --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_node.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_node.json b/lib/docs/db_storage/tests/data/test_node.json new file mode 100644 index 0000000000..7fc72979e0 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_node.json @@ -0,0 +1,8 @@ +{"entries":[ +{"name":"stream.Writable","path":"stream","type":"data","parsed_uri":"/nodejs/stream.writable","anchor":"stream_class_stream_writable","parent_uri":"null","docset":"NodeJS"}, +{"name":"http.IncomingMessage","path":"http","type":"network","parsed_uri":"/nodejs/http.incomingmessage","anchor":"http_http_incomingmessage","parent_uri":"null","docset":"NodeJS"}, +{"name":"EventEmitter","path":"events","type":"event","parsed_uri":"/nodejs/eventemitter","anchor":"events_class_events_eventemitter","parent_uri":"null","docset":"NodeJS"}, +{"name":"buffer.INSPECT_MAX_BYTES","path":"buffer","type":"data","parsed_uri":"/nodejs/buffer.inspect_max_bytes","anchor":"buffer_buffer_inspect_max_bytes","parent_uri":"null","docset":"NodeJS"}, +{"name":"agent.destroy()","path":"http","type":"network","parsed_uri":"/nodejs/agent.destroy","anchor":"http_agent_destroy","parent_uri":"null","docset":"NodeJS"}, +{"name":"new Agent()","path":"http","type":"network","parsed_uri":"/nodejs/new_agent","anchor":"http_new_agent_options","parent_uri":"null","docset":"NodeJS"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_php.html b/lib/docs/db_storage/tests/data/test_php.html new file mode 100644 index 0000000000..3f660a0c04 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_php.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_php.json b/lib/docs/db_storage/tests/data/test_php.json new file mode 100644 index 0000000000..140daf4afb --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_php.json @@ -0,0 +1,14 @@ +{"entries":[ +{"name":"p1","path":"ref.pdo-cubrid.php","type":"others","parsed_uri":"/javascript/p1","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p2","path":"ref.pdo-dblib.php","type":"others","parsed_uri":"/javascript/p2","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p3","path":"ref.pdo-firebird.php","type":"others","parsed_uri":"/javascript/p3","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p4","path":"ref.pdo-ibm.php","type":"others","parsed_uri":"/javascript/p4","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p5","path":"ref.pdo-informix.php","type":"others","parsed_uri":"/javascript/p5","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p6","path":"ref.pdo-mysql.php","type":"others","parsed_uri":"/javascript/p6","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p7","path":"ref.pdo-sqlsrv.php","type":"others","parsed_uri":"/javascript/p7","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p8","path":"ref.pdo-oci.php","type":"others","parsed_uri":"/javascript/p8","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p9","path":"ref.pdo-odbc.php","type":"others","parsed_uri":"/javascript/p9","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p10","path":"ref.pdo-pgsql.php","type":"others","parsed_uri":"/javascript/p10","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p11","path":"ref.pdo-sqlite.php","type":"others","parsed_uri":"/javascript/p11","anchor":"","parent_uri":"null","docset":"PHP"}, +{"name":"p12","path":"ref.pdo-4d.php","type":"others","parsed_uri":"/javascript/p12","anchor":"","parent_uri":"null","docset":"PHP"} +]} diff --git a/lib/docs/db_storage/tests/data/test_postgresql.html b/lib/docs/db_storage/tests/data/test_postgresql.html new file mode 100644 index 0000000000..f445c1d79e --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_postgresql.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_postgresql.json b/lib/docs/db_storage/tests/data/test_postgresql.json new file mode 100644 index 0000000000..a20f569235 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_postgresql.json @@ -0,0 +1,12 @@ +{"entries":[ +{"name":"AND","path":"functions-logical","type":"function","parsed_uri":"/postgresql/function/and","anchor":"657868","parent_uri":"/postgresql/function","docset":"PostgreSQL"}, +{"name":"ANY/SOME (array)","path":"functions-comparisons","type":"function","parsed_uri":"/postgresql/function/anysome_(array)","anchor":"AEN19712","parent_uri":"/postgresql/function","docset":"PostgreSQL"}, +{"name":"Math","path":"functions-math","type":"others","parsed_uri":"/postgresql/math","anchor":"","parent_uri":"null","docset":"PostgreSQL"}, +{"name":"String Functions and Operators","path":"functions-string","type":"string","parsed_uri":"/postgresql/string_functions_and_operators","anchor":"","parent_uri":"/postgresql/string","docset":"PostgreSQL"}, +{"name":"Binary String Functions and Operators","path":"functions-binarystring","type":"operator","parsed_uri":"/postgresql/binary_string_functions_and_operators","anchor":"","parent_uri":"/postgresql/operator","docset":"PostgreSQL"}, +{"name":"Bit String Functions and Operators","path":"functions-bitstring","type":"string","parsed_uri":"/postgresql/bit_string_functions_and_operators","anchor":"","parent_uri":"/postgresql/string","docset":"PostgreSQL"}, +{"name":"LIKE","path":"functions-matching","type":"function","parsed_uri":"/postgresql/function/like","anchor":"FUNCTIONS-LIKE","parent_uri":"/postgresql/function","docset":"PostgreSQL"}, +{"name":"Pattern Matching","path":"functions-matching","type":"function","parsed_uri":"/postgresql/pattern_matching","anchor":"","parent_uri":"/postgresql/function","docset":"PostgreSQL"}, +{"name":"LIKE","path":"functions-matching","type":"function","parsed_uri":"/postgresql/function/like","anchor":"FUNCTIONS-LIKE","parent_uri":"/postgresql/function","docset":"PostgreSQL"}, +{"name":"SIMILAR TO Regular Expressions","path":"functions-matching","type":"function","parsed_uri":"/postgresql/function/similar_to_regular_expressions","anchor":"FUNCTIONS-SIMILARTO-REGEXP","parent_uri":"/postgresql/function","docset":"PostgreSQL"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_python2.html b/lib/docs/db_storage/tests/data/test_python2.html new file mode 100644 index 0000000000..5c712b61eb --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_python2.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_python2.json b/lib/docs/db_storage/tests/data/test_python2.json new file mode 100644 index 0000000000..78f908dcb0 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_python2.json @@ -0,0 +1,7 @@ +{"entries":[ +{"name":"array.array.extend()","path":"library/array","type":"Data Types","parsed_uri":"/python2/array.array.extend","anchor":"array.array.extend","parent_uri":"/python2/array","docset":"Python2"}, +{"name":"array.array.tofile()","path":"library/array","type":"Data Types","parsed_uri":"/python2/array.array.tofile","anchor":"array.array.tofile","parent_uri":"/python2/array","docset":"Python2"}, +{"name":"array.array.fromlist()","path":"library/array","type":"Data Types","parsed_uri":"/python2/array.array.fromlist","anchor":"array.array.fromlist","parent_uri":"/python2/array","docset":"Python2"}, +{"name":"Exceptions","path":"library/exceptions","type":"Built-in Exceptions","parsed_uri":"/python2/exceptions","anchor":"","parent_uri":"null","docset":"Python2"}, +{"name":"array.array.fromunicode()","path":"library/array","type":"Data Types","parsed_uri":"/python2/array.array.fromunicode","anchor":"array.array.fromunicode","parent_uri":"/python2/array","docset":"Python2"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_python3.html b/lib/docs/db_storage/tests/data/test_python3.html new file mode 100644 index 0000000000..02a3bf5ca2 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_python3.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_python3.json b/lib/docs/db_storage/tests/data/test_python3.json new file mode 100644 index 0000000000..1c55ddc037 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_python3.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"Transports and protocols (low-level API)","path":"library/asyncio-protocol","type":"Input/ouput","parsed_uri":"/python3/transports_and_protocols_low-level_api","anchor":"","parent_uri":"null","docset":"Python3"}, +{"name":"Base Event Loop","path":"library/asyncio-eventloop","type":"Input/ouput","parsed_uri":"/python3/base_event_loop","anchor":"","parent_uri":"null","docset":"Python3"}, +{"name":"asyncio.Future","path":"library/asyncio-task","type":"Input/ouput","parsed_uri":"/python3/asyncio.future","anchor":"asyncio.Future","parent_uri":"null","docset":"Python3"}, +{"name":"Event loops","path":"library/asyncio-eventloops","type":"Input/ouput","parsed_uri":"/python3/event_loops","anchor":"","parent_uri":"null","docset":"Python3"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_react.html b/lib/docs/db_storage/tests/data/test_react.html new file mode 100644 index 0000000000..b9665aeeaf --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_react.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_react.json b/lib/docs/db_storage/tests/data/test_react.json new file mode 100644 index 0000000000..f2bd8732f4 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_react.json @@ -0,0 +1,8 @@ +{"entries":[ +{"name":"Animation","path":"animation","type":"others","parsed_uri":"/react/animation","anchor":"","parent_uri":"null","docset":"React"}, +{"name":"Two-Way Binding Helpers","path":"two-way-binding-helpers","type":"others","parsed_uri":"/react/two-way-binding-helpers","anchor":"","parent_uri":"null","docset":"React"}, +{"name":"Class Name Manipulation","path":"class-name-manipulation","type":"others","parsed_uri":"/react/class-name-manipulation","anchor":"","parent_uri":"null","docset":"React"}, +{"name":"React Elements","path":"glossary","type":"others","parsed_uri":"/react/react_elements_","anchor":"react-elements","parent_uri":"null","docset":"React"}, +{"name":"React Components","path":"glossary","type":"others","parsed_uri":"/react/react_components_","anchor":"react-components","parent_uri":"null","docset":"React"}, +{"name":"React Nodes","path":"glossary","type":"others","parsed_uri":"/react/react_nodes_","anchor":"react-nodes","parent_uri":"null","docset":"React"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_redis.html b/lib/docs/db_storage/tests/data/test_redis.html new file mode 100644 index 0000000000..825b0394ac --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_redis.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_redis.json b/lib/docs/db_storage/tests/data/test_redis.json new file mode 100644 index 0000000000..221970c21f --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_redis.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"bitpos","path":"bitpos","type":"function","parsed_uri":"/redis/bitpos","anchor":"","parent_uri":"null","docset":"Redis"}, +{"name":"blpop","path":"blpop","type":"function","parsed_uri":"/redis/blpop","anchor":"","parent_uri":"null","docset":"Redis"}, +{"name":"cluster info","path":"cluster-info","type":"others","parsed_uri":"/redis/cluster-info","anchor":"","parent_uri":"null","docset":"Redis"}, +{"name":"cluster countkeysinslot","path":"cluster-countkeysinslot","type":"others","parsed_uri":"/redis/cluster-countkeysinslot","anchor":"","parent_uri":"null","docset":"Redis"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_requirejs.html b/lib/docs/db_storage/tests/data/test_requirejs.html new file mode 100644 index 0000000000..c278f987a3 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_requirejs.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_requirejs.json b/lib/docs/db_storage/tests/data/test_requirejs.json new file mode 100644 index 0000000000..c224e05490 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_requirejs.json @@ -0,0 +1,7 @@ +{"entries":[ +{"name":"Introduction","path":"jquery.html","type":"guide","parsed_uri":"/requirejs/_introduction__","anchor":"intro","parent_uri":"null","docset":"RequireJS"}, +{"name":"Define a Module with a Name","path":"api.html","type":"api","parsed_uri":"/requirejs/define_a_module_with_a_name","anchor":"modulename","parent_uri":"null","docset":"RequireJS"}, +{"name":"Example loading jquery from a CDN","path":"jquery.html","type":"guide","parsed_uri":"/requirejs/_example_loading_jquery_from_a_cdn__","anchor":"cdnconfig","parent_uri":"null","docset":"RequireJS"}, +{"name":"Mapping Modules to use noConflict","path":"jquery.html","type":"guide","parsed_uri":"/requirejs/_mapping_modules_to_use_noconflict__","anchor":"noconflictmap","parent_uri":"null","docset":"RequireJS"}, +{"name":"map","path":"api.html","type":"configuration","parsed_uri":"/requirejs/map","anchor":"config-map","parent_uri":"null","docset":"RequireJS"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_sass.html b/lib/docs/db_storage/tests/data/test_sass.html new file mode 100644 index 0000000000..c5d6f45a41 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_sass.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_sass.json b/lib/docs/db_storage/tests/data/test_sass.json new file mode 100644 index 0000000000..6176073f08 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_sass.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"hsla","path":"sass/script/functions","type":"function","parsed_uri":"/sass/hsla","anchor":"hsla-instance_method","parent_uri":"null","docset":"Sass"}, +{"name":"adjust_hue","path":"sass/script/functions","type":"function","parsed_uri":"/sass/adjust_hue","anchor":"adjust_hue-instance_method","parent_uri":"null","docset":"Sass"}, +{"name":"red","path":"sass/script/functions","type":"function","parsed_uri":"/sass/red","anchor":"red-instance_method","parent_uri":"null","docset":"Sass"}, +{"name":":load_paths","path":"file.sass_reference","type":"styles","parsed_uri":"/sass/-load_paths","anchor":"load_paths-option","parent_uri":"null","docset":"Sass"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_sinon.html b/lib/docs/db_storage/tests/data/test_sinon.html new file mode 100644 index 0000000000..02ef0f9d37 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_sinon.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_sinon.json b/lib/docs/db_storage/tests/data/test_sinon.json new file mode 100644 index 0000000000..2a321c2618 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_sinon.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"Assertions","path":"index","type":"others","parsed_uri":"/sinonjs/assertions","anchor":"assertions","parent_uri":"null","docset":"SinonJS"}, +{"name":"clock.restore()","path":"index","type":"function","parsed_uri":"/sinonjs/clock.restore","anchor":"clock-restore","parent_uri":"null","docset":"SinonJS"}, +{"name":"clock.tick()","path":"index","type":"function","parsed_uri":"/sinonjs/clock.tick","anchor":"clock-tick","parent_uri":"null","docset":"SinonJS"}, +{"name":"expectation.atLeast()","path":"index","type":"others","parsed_uri":"/sinonjs/expectation.atleast","anchor":"expectation-atleast","parent_uri":"null","docset":"SinonJS"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_socketio.html b/lib/docs/db_storage/tests/data/test_socketio.html new file mode 100644 index 0000000000..c92bf5d8cc --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_socketio.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_socketio.json b/lib/docs/db_storage/tests/data/test_socketio.json new file mode 100644 index 0000000000..d2aebf21ec --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_socketio.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"Namespace.use","path":"server-api","type":"namespace","parsed_uri":"/socketio/server_api/namespace.use","anchor":"namespace-use-fn-function-namespace","parent_uri":"/socketio/server_api","docset":"SocketIO"},{"name":"Namespaces","path":"rooms-and-namespaces","type":"guide","parsed_uri":"/socketio/namespaces","anchor":"","parent_uri":"null","docset":"SocketIO"}, +{"name":"Namespace","path":"server-api","type":"namespace","parsed_uri":"/socketio/server_api/namespace","anchor":"namespace","parent_uri":"/socketio/server_api","docset":"SocketIO"}, +{"name":"Server API","path":"server-api","type":"guide","parsed_uri":"/socketio/server_api","anchor":"","parent_uri":"null","docset":"SocketIO"}, +{"name":"Server.emit","path":"server-api","type":"network","parsed_uri":"/socketio/server_api/server.emit","anchor":"server-emit","parent_uri":"/socketio/server_api","docset":"SocketIO"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_svg.html b/lib/docs/db_storage/tests/data/test_svg.html new file mode 100644 index 0000000000..be5cfd4887 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_svg.html @@ -0,0 +1 @@ +

Global attributes

\ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_svg.json b/lib/docs/db_storage/tests/data/test_svg.json new file mode 100644 index 0000000000..18125d91f3 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_svg.json @@ -0,0 +1,8 @@ +{"entries":[ +{"name":"attribute","path":"attribute","type":"attribute","parsed_uri":"/svg/attribute","anchor":"","parent_uri":"null","docset":"SVG"}, +{"name":"attribute.class","path":"attribute/class","type":"attribute","parsed_uri":"/svg/attribute/class","anchor":"","parent_uri":"/svg/attribute","docset":"SVG"}, +{"name":"attribute.style","path":"attribute/style","type":"attribute","parsed_uri":"/svg/attribute/style","anchor":"","parent_uri":"/svg/attribute","docset":"SVG"}, +{"name":"attribute.externalresourcesrequired","path":"attribute/externalresourcesrequired","type":"attribute","parsed_uri":"/svg/attribute/externalresourcesrequired","anchor":"","parent_uri":"/svg/attribute","docset":"SVG"}, +{"name":"attribute.dx","path":"attribute/dx","type":"attribute","parsed_uri":"/svg/attribute/dx","anchor":"","parent_uri":"/svg/attribute","docset":"SVG"}, +{"name":"attribute.dy","path":"attribute/dy","type":"attribute","parsed_uri":"/svg/attribute/dy","anchor":"","parent_uri":"/svg/attribute","docset":"SVG"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_underscore.html b/lib/docs/db_storage/tests/data/test_underscore.html new file mode 100644 index 0000000000..5eda458df8 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_underscore.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_underscore.json b/lib/docs/db_storage/tests/data/test_underscore.json new file mode 100644 index 0000000000..2f27f7beb5 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_underscore.json @@ -0,0 +1,6 @@ +{"entries":[ +{"name":"isEmpty","path":"index","type":"object","parsed_uri":"/underscorejs/isempty","anchor":"isEmpty","parent_uri":"null","docset":"UnderscoreJS"}, +{"name":"groupBy","path":"index","type":"collection","parsed_uri":"/underscorejs/groupby","anchor":"groupBy","parent_uri":"null","docset":"UnderscoreJS"}, +{"name":"zip","path":"index","type":"array","parsed_uri":"/underscorejs/zip","anchor":"zip","parent_uri":"null","docset":"UnderscoreJS"}, +{"name":"findIndex","path":"index","type":"array","parsed_uri":"/underscorejs/findindex","anchor":"findIndex","parent_uri":"null","docset":"UnderscoreJS"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_xpath.html b/lib/docs/db_storage/tests/data/test_xpath.html new file mode 100644 index 0000000000..0f62b94f4a --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_xpath.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_xpath.json b/lib/docs/db_storage/tests/data/test_xpath.json new file mode 100644 index 0000000000..dff21f0ce8 --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_xpath.json @@ -0,0 +1,8 @@ +{"entries":[ +{"name":"boolean()","path":"functions/boolean","type":"function","parsed_uri":"/xpath/functions/boolean","anchor":"","parent_uri":"/xpath/functions","docset":"XPath"}, +{"name":"ceiling()","path":"functions/ceiling","type":"function","parsed_uri":"/xpath/functions/ceiling","anchor":"","parent_uri":"/xpath/functions","docset":"XPath"}, +{"name":"choose()","path":"functions/choose","type":"function","parsed_uri":"/xpath/functions/choose","anchor":"","parent_uri":"/xpath/functions","docset":"XPath"}, +{"name":"concat()","path":"functions/concat","type":"function","parsed_uri":"/xpath/functions/concat","anchor":"","parent_uri":"/xpath/functions","docset":"XPath"}, +{"name":"contains()","path":"functions/contains","type":"function","parsed_uri":"/xpath/functions/contains","anchor":"","parent_uri":"/xpath/functions","docset":"XPath"}, +{"name":"count()","path":"functions/count","type":"function","parsed_uri":"/xpath/functions/count","anchor":"","parent_uri":"/xpath/functions","docset":"XPath"} +]} \ No newline at end of file diff --git a/lib/docs/db_storage/tests/data/test_yii.html b/lib/docs/db_storage/tests/data/test_yii.html new file mode 100644 index 0000000000..7186d8c17b --- /dev/null +++ b/lib/docs/db_storage/tests/data/test_yii.html @@ -0,0 +1 @@ +