v2.5.1-beta

* Required for Tautulli/Tautulli-Issues#255

.dockerignore

@@ -3,6 +3,8 @@
 .gitignore
 contrib
 init-scripts
+package
 pylintrc
 *.md
 !CHANGELOG*.md
+start.bat

.github/workflows/publish-docker.yml

@@ -1,12 +1,15 @@
 name: Publish Docker
 on:
   push:
-    branches: [master, beta, nightly]
+    branches: [master, beta, nightly, python3]
     tags: [v*]
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
       - name: Prepare
         id: prepare
         run: |
@@ -25,20 +28,28 @@ jobs:
           echo ::set-output name=commit::${GITHUB_SHA}
           echo ::set-output name=build_date::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
           echo ::set-output name=docker_platforms::linux/amd64,linux/arm64,linux/arm
-          echo ::set-output name=docker_image::tautulli/tautulli
+          echo ::set-output name=docker_image::${{ secrets.DOCKER_REPO }}/tautulli
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: crazy-max/ghaction-docker-buildx@v1
+        uses: crazy-max/ghaction-docker-buildx@v3
         with:
-          version: latest
+          buildx-version: latest
 
-      - name: Checkout
-        uses: actions/checkout@v2
+      - name: Cache Docker Layers
+        id: cache
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
 
       - name: Docker Buildx (no push)
         run: |
           docker buildx build \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --cache-to "type=local,dest=/tmp/.buildx-cache" \
             --platform ${{ steps.prepare.outputs.docker_platforms }} \
             --output "type=image,push=false" \
             --build-arg "TAG=${{ steps.prepare.outputs.tag }}" \
@@ -59,6 +70,7 @@ jobs:
         if: success()
         run: |
           docker buildx build \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
             --platform ${{ steps.prepare.outputs.docker_platforms }} \
             --output "type=image,push=true" \
             --build-arg "TAG=${{ steps.prepare.outputs.tag }}" \
@@ -79,5 +91,5 @@ jobs:
         with:
           webhook: ${{ secrets.DISCORD_WEBHOOK }}
           status: ${{ job.status }}
-          job: ${{ github.workflow }}
+          title: ${{ github.workflow }}
           nofail: true

.github/workflows/publish-release.yml

@@ -1,28 +1,204 @@
 name: Publish Release
 on:
   push:
+    branches: [master, beta, nightly, python3]
     tags: [v*]
+
 jobs:
-  build:
-    runs-on: ubuntu-latest
+  build-windows:
+    runs-on: windows-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@master
-      - name: Get Release Version
-        run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+        uses: actions/checkout@v2
+
+      - name: Set Release Version
+        id: get_version
+        shell: bash
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION_NSIS=${GITHUB_REF#refs/tags/v}.1
+            echo ::set-output name=VERSION_NSIS::${VERSION_NSIS/%-beta.1/.0}
+            echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/v}
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+          else
+            echo ::set-output name=VERSION_NSIS::0.0.0.0
+            echo ::set-output name=VERSION::0.0.0
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_SHA::7}
+          fi
+          echo $GITHUB_SHA > version.txt
+
+      - name: Set Up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Cache Dependencies
+        id: cache_dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{ runner.os }}-pip-${{ hashFiles('package/requirements-windows.txt') }}
+          restore-keys: ${{ runner.os }}-pip-
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r package/requirements-windows.txt
+
+      - name: Build Package
+        run: |
+          pyinstaller -y ./package/Tautulli-windows.spec
+
+      - name: Create Installer
+        uses: joncloud/makensis-action@v1.2
+        with:
+          script-file: ./package/Tautulli.nsi
+          arguments: /DVERSION=${{ steps.get_version.outputs.VERSION_NSIS }} /DINSTALLER_NAME=..\Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}.exe
+          include-more-plugins: true
+          include-custom-plugins-path: package/nsis-plugins
+
+      - name: Upload Installer
+        uses: actions/upload-artifact@v2
+        with:
+          name: Tautulli-windows-installer
+          path: Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}.exe
+
+      - name: Post Status to Discord
+        uses: sarisia/actions-status-discord@v1
+        if: always()
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          status: ${{ job.status }}
+          title: Build Windows Installer
+          nofail: true
+
+  build-macos:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Set Release Version
+        id: get_version
+        shell: bash
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo ::set-env name=VERSION::${GITHUB_REF#refs/tags/v}
+            echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/v}
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+          else
+            echo ::set-env name=VERSION::0.0.0
+            echo ::set-output name=VERSION::0.0.0
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_SHA::7}
+          fi
+          echo $GITHUB_SHA > version.txt
+
+      - name: Set Up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Cache Dependencies
+        id: cache_dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/Library/Caches/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('package/requirements-macos.txt') }}
+          restore-keys: ${{ runner.os }}-pip-
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r package/requirements-macos.txt
+
+      - name: Build Package
+        run: |
+          pyinstaller -y ./package/Tautulli-macos.spec
+
+      - name: Create Installer
+        run: |
+          sudo pkgbuild --install-location /Applications --version ${{ steps.get_version.outputs.VERSION }} --component ./dist/Tautulli.app --scripts ./package/macos-scripts Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}.pkg
+
+      - name: Upload Installer
+        uses: actions/upload-artifact@v2
+        with:
+          name: Tautulli-macos-installer
+          path: Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}.pkg
+
+      - name: Post Status to Discord
+        uses: sarisia/actions-status-discord@v1
+        if: always()
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          status: ${{ job.status }}
+          title: Build MacOS Installer
+          nofail: true
+
+  release:
+    needs: [build-windows, build-macos]
+    if: startsWith(github.ref, 'refs/tags/') && always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get Build Job Status
+        uses: technote-space/workflow-conclusion-action@v1
+
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Set Release Version
+        id: get_version
+        run: |
+          echo ::set-output name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+
+      - name: Download Windows Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/download-artifact@v2
+        with:
+          name: Tautulli-windows-installer
+
+      - name: Download MacOS Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/download-artifact@v2
+        with:
+          name: Tautulli-macos-installer
+
       - name: Get Changelog
-        run: echo ::set-env name=CHANGELOG::"$( sed -n '/^## /{p; :loop n; p; /^## /q; b loop}' CHANGELOG.md | sed '$d' | sed '$d' | sed '$d' | sed ':a;N;$!ba;s/\n/%0A/g' )"
+        id: get_changelog
+        run: echo ::set-output name=CHANGELOG::"$( sed -n '/^## /{p; :loop n; p; /^## /q; b loop}' CHANGELOG.md | sed '$d' | sed '$d' | sed '$d' | sed ':a;N;$!ba;s/\n/%0A/g' )"
+
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          tag_name: ${{ env.RELEASE_VERSION }}
-          release_name: Tautulli ${{ env.RELEASE_VERSION }}
+          tag_name: ${{ steps.get_version.outputs.RELEASE_VERSION }}
+          release_name: Tautulli ${{ steps.get_version.outputs.RELEASE_VERSION }}
           body: |
             ## Changelog
             
-            ##${{ env.CHANGELOG }}
+            ##${{ steps.get_changelog.outputs.CHANGELOG }}
           draft: false
-          prerelease: ${{ endsWith(env.RELEASE_VERSION, '-beta') }}
+          prerelease: ${{ endsWith(steps.get_version.outputs.RELEASE_VERSION, '-beta') }}
+
+      - name: Upload Windows Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}.exe
+          asset_name: Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}.exe
+          asset_content_type: application/vnd.microsoft.portable-executable
+
+      - name: Upload MacOS Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}.pkg
+          asset_name: Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}.pkg
+          asset_content_type: application/vnd.apple.installer+xml

.gitignore

@@ -19,6 +19,8 @@ backups/*
 cache/*
 newsletters/*
 *.mmdb
+version.txt
+branch.txt
 
 # HTTPS Cert/Key #
 ##################
@@ -74,3 +76,7 @@ _ReSharper*/
 /logs
 .project
 .pydevproject
+
+#Ignore files generated by pyinstaller
+/build
+/dist

API.md

@@ -672,7 +672,7 @@ Returns:
 
 
 ### get_geoip_lookup
-Get the geolocation info for an IP address. The GeoLite2 database must be installed.
+Get the geolocation info for an IP address.
 
 ```
 Required parameters:
@@ -683,7 +683,7 @@ Optional parameters:
 
 Returns:
     json:
-        {"continent": "North America",
+        {"code": 'US",
          "country": "United States",
          "region": "California",
          "city": "Mountain View",
@@ -693,9 +693,6 @@ Returns:
          "longitude": -122.0838,
          "accuracy": 1000
          }
-    json:
-        {"error": "The address 127.0.0.1 is not in the database."
-         }
 ```
 
 
@@ -2558,26 +2555,27 @@ Returns:
 
 
 ### import_database
-Import a PlexWatch or Plexivity database into Tautulli.
+Import a Tautulli, PlexWatch, or Plexivity database into Tautulli.
 
 ```
 Required parameters:
-    app (str):                      "plexwatch" or "plexivity"
+    app (str):                      "tautulli" or "plexwatch" or "plexivity"
     database_path (str):            The full path to the plexwatch database file
-    table_name (str):               "processed" or "grouped"
+    method (str):                   For Tautulli only, "merge" or "overwrite"
+    table_name (str):               For PlexWatch or Plexivity only, "processed" or "grouped"
+
 
 Optional parameters:
-    import_ignore_interval (int):   The minimum number of seconds for a stream to import
+    backup (bool):                  For Tautulli only, true or false whether to backup
+                                    the current database before importing
+    import_ignore_interval (int):   For PlexWatch or Plexivity only, the minimum number
+                                    of seconds for a stream to import
 
 Returns:
     None
 ```
 
 
-### install_geoip_db
-Downloads and installs the GeoLite2 database
-
-
 ### notify
 Send a notification using Tautulli.
 
@@ -2854,10 +2852,6 @@ Returns:
 ```
 
 
-### uninstall_geoip_db
-Uninstalls the GeoLite2 database
-
-
 ### update
 Update Tautulli.
 

CHANGELOG.md

@@ -1,31 +1,87 @@
 # Changelog
 
-## v2.2.2-beta (2020-04-12)
+## v2.5.1-beta (2020-06-26)
 
 * Notifications:
+  * Fix: Error uploading images to Cloudinary on Python 2.
+  * Fix: Testing browser notifications alert not disappearing.
+  * Change: Default recently added notification delay set to 300 seconds.
+* UI: 
+  * Fix: MacOS menu bar icon causing Tautulli to fail to start.
+  * New: Added platform icon for LG devices.
+* Mobile App:
+  * Fix: Improved API security and validation when registering the Android app.
+* Other:
+  * Fix: Error creating self-signed certificates on Python 3.
+  * Fix: Docker container not respecting the PUID and PGID environment variables.
+  * Fix: Tautulli login session cookie not set on the HTTP root path.
+  * Remove: Ability to login to Tautulli using a Plex username and password has been removed. Login using a Plex.tv account is only supported via OAuth.
+
+
+## v2.5.0-beta (2020-05-31)
+
+* Announcements:
+  * Tautulli now supports Python 3!
+  * Python 2 is still supported for the time being, but it is recommended to upgrade to Python 3.
+* UI:
+  * New: Windows and MacOS setting to enable Tautulli to start automatically when you login.
+  * New: Added menu bar icon for MacOS.
+  * New: Ability to import a Tautulli database in the settings.
+  * New: Added Tautulli news area on the settings page.
+* Other:
+  * New: Windows and MacOS app installers to install Tautulli without needing Python installed.
+
+
+## v2.2.4 (2020-05-16)
+
+* Monitoring:
+  * Fix: Show "None" as the subtitle source on the activity card for user selected subtitles.
+* UI:
+  * Fix: Deleted libraries were showing up on the homepage library cards.
+  * Fix: Libraries could get stuck as inactive in the database in some instances.
+* API:
+  * Fix: Incorrect title was being returned for the get_history API command.
+* Other:
+  * Fix: Plex remote access check was not being rescheduled after changing the settings.
+
+
+## v2.2.3 (2020-05-01)
+
+* Notifications:
+  * Fix: Notification grouping by season/album and show/artist not enabled by default.
+  * Fix: The rating key notification parameter was being overwritten when 3rd party lookup was enabled.
+  * Fix: Missing artist value for Musicbrainz lookup in certain situations. 
   * New: Added notification trigger for Tautulli database corruption.
   * New: Added TAUTULLI_PYTHON_VERSION to script notification environment variables.
-  * Fix: Notification grouping by season/album and show/artist not enabled by default.
+  * New: Added Plex Android / iOS App notification agent.
+  * New: Added bandwidth notification parameters.
+  * New: Added user thumb to notification parameters.
+  * New: Added initial stream notification parameter and threshold setting to determine if a stream is the first stream of a continuous streaming session.
+  * New: Added Plex remote access notification parameters.
   * Change: The file size notification parameter is now reported in SI units. (Thanks @aaronldunlap)
 * UI:
   * Fix: Delete lookup info from the media info page failing.
+  * Fix: XBMC platform icon not being redirected to the Kodi platform icon.
+  * Fix: History table was not being refreshed after deleting entries.
   * New: Added icon on the users table to indicate if the user is not on the Plex server.
   * New: Added icon on the libraries table to indicate if the library is not on the Plex server.
-  * Fix: XBMC platform icon not being redirected to the Kodi platform icon.
   * Change: Improved deleting libraries so libraries with the same section ID are not also deleted.
+* Mobile App:
+  * Fix: Temporary device token was not being invalidated after cancelling device registration.
 * API:
-  * Fix: Returning XML for the API failing due to unicode characters.
+  * Fix: Returning XML from the API failing due to unicode characters.
   * Fix: Grouping parameter for various API commands not falling back to default setting.
   * New: Added time_queries parameter to get_library_watch_time_stats and get_user_watch_time_stats API command. (Thanks @KaasKop97)
   * New: Added an "is_active" return value to the get_user, get_users, get_library, and get_libraries API commands which indicates if the user or library is on the Plex server.
   * New: Added delete_history API command.
   * Change: Added optional parameter for row_ids for delete_library, delete_user, delete_all_library_history, and delete_all_user_history API commands.
-* Mobile App:
-  * Fix: Temporary device token not being invalidated after cancelling device registration.
 * Other:
   * Fix: Update failing on CentOS due to an older git version.
   * Fix: Manifest file for creating a web app had incorrect info.
+  * Fix: Auto-updater was not scheduled when enabling the setting unless Tautulli was restarted.
   * New: Docker images updated to support ARM platforms.
+  * Change: Remove the unnecessary optional Plex logs volume from the Docker image.
+  * Change: Use Plex.tv for GeoIP lookup instead of requiring the MaxMind GeoLite2 database.
 
 
 ## v2.2.1 (2020-03-28)

CONTRIBUTING.md

@@ -9,7 +9,7 @@ All pull requests should be based on the `nightly` branch, to minimize cross mer
 ### Python Code
 
 #### Compatibility
-The code should work with Python 2.7. Note that Tautulli runs on many different platforms.
+The code should work with Python 2.7.17 or Python 3.6+. Note that Tautulli runs on many different platforms.
 
 Re-use existing code. Do not hesitate to add logging in your code. You can the logger module `plexpy.logger.*` for this. Web requests are invoked via `plexpy.request.*` and derived ones. Use these methods to automatically add proper and meaningful error handling.
 
@@ -38,4 +38,4 @@ HTML5 compatible browsers are targeted.
 * 4 space indentation
 * `methodName`
 * `variableName`
-* `ClassName`
+* `ClassName`

Dockerfile

@@ -1,4 +1,4 @@
-FROM tautulli/tautulli-baseimage:latest
+FROM tautulli/tautulli-baseimage:python3
 
 LABEL maintainer="Tautulli"
 
@@ -16,8 +16,8 @@ RUN \
 
 COPY . /app
 
-CMD [ "python", "Tautulli.py", "--datadir", "/config" ]
+ENTRYPOINT [ "./start.sh" ]
 
-VOLUME /config /plex_logs
+VOLUME /config
 EXPOSE 8181
 HEALTHCHECK  --start-period=90s CMD curl -ILfSs http://localhost:8181/status > /dev/null || curl -ILfkSs https://localhost:8181/status > /dev/null || exit 1

PlexPy.py

@@ -1,8 +1,4 @@
-#!/bin/sh
-''''which python    >/dev/null 2>&1 && exec python    "$0" "$@" # '''
-''''which python2   >/dev/null 2>&1 && exec python2   "$0" "$@" # '''
-''''which python2.7 >/dev/null 2>&1 && exec python2.7 "$0" "$@" # '''
-''''exec echo "Error: Python not found!" # '''
+#!/usr/bin/env python
 
 # -*- coding: utf-8 -*-
 

README.md

@@ -29,14 +29,15 @@ This project is based on code from [Headphones](https://github.com/rembo10/headp
 
 ## Installation & Support
 
-[![Python](https://img.shields.io/badge/python-v2.7.17-blue?style=flat-square)](https://python.org/downloads/release/python-2717/)
+[![Python](https://img.shields.io/badge/python-2.7.17,%203.6,%203.7,%203.8-blue?style=flat-square)](https://python.org/downloads)
 [![Docker Pulls](https://img.shields.io/docker/pulls/tautulli/tautulli?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli)
 [![Docker Stars](https://img.shields.io/docker/stars/tautulli/tautulli?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli)
 
 | Status | Branch: `master` | Branch: `beta` | Branch: `nightly` |
 | --- | --- | --- | --- |
-| Release | [![Release@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![Release Date@master](https://img.shields.io/github/release-date/Tautulli/Tautulli?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/releases/latest) | [![Release@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![Commits@nightly](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/beta?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/beta) | [![Last Commits@nightly](https://img.shields.io/github/last-commit/Tautulli/Tautulli/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) <br> [![Commits@nightly](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) |
-| Docker  | [![Docker@master](https://img.shields.io/badge/tautulli-tautulli:latest-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/master?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Amaster) | [![Docker@beta](https://img.shields.io/badge/tautulli-tautulli:beta-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/beta?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Abeta) | [![Docker@nightly](https://img.shields.io/badge/tautulli-tautulli:nightly-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/nightly?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Anightly) |
+| Release   | [![Release@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![Release Date@master](https://img.shields.io/github/release-date/Tautulli/Tautulli?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/releases/latest) | [![Release@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![Commits@beta](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/beta?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/beta) | [![Last Commits@nightly](https://img.shields.io/github/last-commit/Tautulli/Tautulli/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) <br> [![Commits@nightly](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) |
+| Docker    | [![Docker@master](https://img.shields.io/badge/docker-latest-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/master?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Amaster) | [![Docker@beta](https://img.shields.io/badge/docker-beta-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@beta](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/beta?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Abeta) | [![Docker@nightly](https://img.shields.io/badge/docker-nightly-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@nightly](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/nightly?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Anightly) |
+| Installer | [![Windows@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=windows&style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![MacOS@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=macos&style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![Installer Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Release/master?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Release"+branch%3Amaster) | [![Windows@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=windows&include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![MacOS@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=macos&include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![Installer Build@beta](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Release/beta?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Release"+branch%3Abeta) | [![Installer Build@nightly](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Release/nightly?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Release"+branch%3Anightly) |
 
 [![Wiki](https://img.shields.io/badge/github-wiki-black?style=flat-square)](https://github.com/Tautulli/Tautulli-Wiki/wiki)
 [![Discord](https://img.shields.io/discord/183396325142822912?label=discord&style=flat-square&color=7289DA)](https://tautulli.com/discord)

Tautulli.py

@@ -1,8 +1,4 @@
-#!/bin/sh
-''''which python    >/dev/null 2>&1 && exec python    "$0" "$@" # '''
-''''which python2   >/dev/null 2>&1 && exec python2   "$0" "$@" # '''
-''''which python2.7 >/dev/null 2>&1 && exec python2.7 "$0" "$@" # '''
-''''exec echo "Error: Python not found!" # '''
+#!/usr/bin/env python
 
 # -*- coding: utf-8 -*-
 
@@ -27,17 +23,24 @@ import sys
 # Ensure lib added to path, before any other imports
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib'))
 
+from future.builtins import str
+
+import appdirs
 import argparse
 import datetime
 import locale
 import pytz
 import signal
 import time
+import threading
 import tzlocal
 
 import plexpy
-from plexpy import config, database, helpers, logger, webstart
-
+from plexpy import common, config, database, helpers, logger, webstart
+if common.PLATFORM == 'Windows':
+    from plexpy import windows
+elif common.PLATFORM == 'Darwin':
+    from plexpy import macos
 
 # Register signals, such as CTRL + C
 signal.signal(signal.SIGINT, plexpy.sig_handler)
@@ -51,12 +54,14 @@ def main():
     """
 
     # Fixed paths to Tautulli
-    if hasattr(sys, 'frozen'):
+    if hasattr(sys, 'frozen') and hasattr(sys, '_MEIPASS'):
+        plexpy.FROZEN = True
         plexpy.FULL_PATH = os.path.abspath(sys.executable)
+        plexpy.PROG_DIR = sys._MEIPASS
     else:
         plexpy.FULL_PATH = os.path.abspath(__file__)
+        plexpy.PROG_DIR = os.path.dirname(plexpy.FULL_PATH)
 
-    plexpy.PROG_DIR = os.path.dirname(plexpy.FULL_PATH)
     plexpy.ARGS = sys.argv[1:]
 
     # From sickbeard
@@ -122,7 +127,7 @@ def main():
 
     if args.dev:
         plexpy.DEV = True
-        logger.debug(u"Tautulli is running in the dev environment.")
+        logger.debug("Tautulli is running in the dev environment.")
 
     if args.daemon:
         if sys.platform == 'win32':
@@ -175,6 +180,8 @@ def main():
     # Determine which data directory and config file to use
     if args.datadir:
         plexpy.DATA_DIR = args.datadir
+    elif plexpy.FROZEN:
+        plexpy.DATA_DIR = appdirs.user_data_dir("Tautulli", False)
     else:
         plexpy.DATA_DIR = plexpy.PROG_DIR
 
@@ -229,25 +236,48 @@ def main():
         try:
             import OpenSSL
         except ImportError:
-            logger.warn("The pyOpenSSL module is missing. Install this " \
+            logger.warn("The pyOpenSSL module is missing. Install this "
                         "module to enable HTTPS. HTTPS will be disabled.")
             plexpy.CONFIG.ENABLE_HTTPS = False
 
     # Try to start the server. Will exit here is address is already in use.
     webstart.start()
 
-    # Windows system tray icon
-    if os.name == 'nt' and plexpy.CONFIG.WIN_SYS_TRAY:
-        plexpy.win_system_tray()
-
-    logger.info("Tautulli is ready!")
+    if common.PLATFORM == 'Windows':
+        if plexpy.CONFIG.SYS_TRAY_ICON:
+            plexpy.WIN_SYS_TRAY_ICON = windows.WindowsSystemTray()
+            plexpy.WIN_SYS_TRAY_ICON.start()
+        windows.set_startup()
+    elif common.PLATFORM == 'Darwin':
+        macos.set_startup()
 
     # Open webbrowser
     if plexpy.CONFIG.LAUNCH_BROWSER and not args.nolaunch and not plexpy.DEV:
         plexpy.launch_browser(plexpy.CONFIG.HTTP_HOST, plexpy.HTTP_PORT,
                               plexpy.HTTP_ROOT)
 
-    # Wait endlessy for a signal to happen
+    if common.PLATFORM == 'Darwin' and plexpy.CONFIG.SYS_TRAY_ICON:
+        if not macos.HAS_PYOBJC:
+            logger.warn("The pyobjc module is missing. Install this "
+                        "module to enable the MacOS menu bar icon.")
+            plexpy.CONFIG.SYS_TRAY_ICON = False
+
+        if plexpy.CONFIG.SYS_TRAY_ICON:
+            # MacOS menu bar icon must be run on the main thread and is blocking
+            # Start the rest of Tautulli on a new thread
+            threading.Thread(target=wait).start()
+            plexpy.MAC_SYS_TRAY_ICON = macos.MacOSSystemTray()
+            plexpy.MAC_SYS_TRAY_ICON.start()
+        else:
+            wait()
+    else:
+        wait()
+
+
+def wait():
+    logger.info("Tautulli is ready!")
+
+    # Wait endlessly for a signal to happen
     while True:
         if not plexpy.SIGNAL:
             try:
@@ -265,11 +295,14 @@ def main():
                 plexpy.shutdown(restart=True, checkout=True)
             elif plexpy.SIGNAL == 'reset':
                 plexpy.shutdown(restart=True, reset=True)
-            else:
+            elif plexpy.SIGNAL == 'update':
                 plexpy.shutdown(restart=True, update=True)
+            else:
+                logger.error('Unknown signal. Shutting down...')
+                plexpy.shutdown()
 
             plexpy.SIGNAL = None
 
-# Call main()
+
 if __name__ == "__main__":
     main()

data/interfaces/default/app_import.html

@@ -5,46 +5,97 @@
             <h4 class="modal-title">Import ${app} Database</h4>
         </div>
         <div class="modal-body" id="modal-text">
-            <p class="help-block">
-                <%
-                    v = ''
-                    if app == 'PlexWatch':
-                        v = '0.3.2'
-                    elif app == 'Plexivity':
-                        v = '0.9.8'
-                %>
-                <strong>Please ensure your ${app} database is at version ${v} or higher.</strong>
-            </p>
-            <div class="form-group">
-                <label for="db_location">Database Location</label>
-                <div class="row">
-                    <div class="col-xs-8">
-                        <input type="text" class="form-control" id="db_location" name="db_location" value="" required>
+            <form id="import_database" enctype="multipart/form-data" method="post" name="import_database">
+                <input type="hidden" id="import_app" name="import_app" value="${app.lower()}" />
+                % if app in ('PlexWatch', 'Plexivity'):
+                <p class="help-block">
+                    <%
+                        v = ''
+                        if app == 'PlexWatch':
+                            v = '0.3.2'
+                        elif app == 'Plexivity':
+                            v = '0.9.8'
+                    %>
+                    <strong>Please ensure your ${app} database is at version ${v} or higher.</strong>
+                </p>
+                % endif
+                <div class="form-group">
+                    <label for="import_database_file">Option 1: Upload a Database File</label>
+                    <div class="row">
+                        <div class="col-xs-12">
+                            <div class="input-group">
+                                <label for="import_database_file" class="input-group-btn">
+                                    <span class="btn btn-form">Upload</span>
+                                    <input type="file" style="display: none;" id="import_database_file" name="import_database_file" required>
+                                </label>
+                                <input id="import_database_file_name" type="text" class="form-control" disabled>
+                            </div>
+                        </div>
                     </div>
+                    <p class="help-block">Upload the ${app} database you wish to import.</p>
                 </div>
-                <p class="help-block">Enter the path and file name for the ${app} database you wish to import.</p>
-            </div>
-            <div class="form-group">
-                <label for="table_name">Table Name</label>
-                <div class="row">
-                    <div class="col-xs-4">
-                        <select id="table_name" class="form-control" name="table_name">
-                            <option value="processed">processed</option>
-                            <option value="grouped">grouped</option>
-                        </select>
+                <div class="form-group">
+                    <label for="import_database_path">Option 2: Browse for a Database File</label>
+                    <div class="row">
+                        <div class="col-xs-12">
+                            <div class="input-group">
+                                <span class="input-group-btn">
+                                    <button class="btn btn-form" type="button" id="import_database_path_browse">Browse</button>
+                                </span>
+                                <input type="text" class="form-control" id="import_database_path" name="import_database_path" value="" required disabled>
+                            </div>
+                        </div>
                     </div>
+                    <p class="help-block">Browse for the ${app} database you wish to import.</p>
                 </div>
-                <p class="help-block">The table name from which you wish to import. Only import one of these, importing both will result in duplicated data.</p>
-            </div>
-            <div class="form-group">
-                <label for="import_ignore_interval">Ignore Interval</label>
-                <div class="row">
-                    <div class="col-xs-2">
-                        <input type="text" class="form-control" id="import_ignore_interval" name="import_ignore_interval" value="120" required>
+                % if app == 'Tautulli':
+                <div class="form-group">
+                    <label for="table_name">Import Method</label>
+                    <div class="row">
+                        <div class="col-xs-4">
+                            <select class="form-control" id="import_method" name="import_method">
+                                <option value="merge">Merge</option>
+                                <option value="overwrite">Overwrite</option>
+                            </select>
+                        </div>
                     </div>
+                    <p class="help-block">Select how you would like to import the Tautulli history.</p>
+                    <ul class="help-block" style="padding-inline-start: 15px;">
+                        <li><strong>Merge</strong> will add all history and remove any duplicates from the imported database into the current database.</li>
+                        <li><strong>Overwrite</strong> will replace all history in the current database with the imported database.</li>
+                    </ul>
+                    <p class="help-block">Note: Libraries, users, notification agents, newsletter agents, and registered mobile devices will also be imported</p>
                 </div>
-                <p class="help-block">Enter the minimum duration (in seconds) an item must have been active for. Set to 0 to import all.</p>
-            </div>
+                <div class="checkbox">
+                    <label>
+                        <input type="checkbox" name="import_backup_db" id="import_backup_db" value="1" checked> Backup Current Database
+                    </label>
+                    <p class="help-block">Automatically create a backup of the current database before importing.</p>
+                </div>
+                % else:
+                <div class="form-group">
+                    <label for="import_table_name">Table Name</label>
+                    <div class="row">
+                        <div class="col-xs-4">
+                            <select class="form-control" id="import_table_name" name="import_table_name">
+                                <option value="processed">Processed</option>
+                                <option value="grouped">Grouped</option>
+                            </select>
+                        </div>
+                    </div>
+                    <p class="help-block">Select the table name from which you wish to import. Only import one of these, importing both will result in duplicated data.</p>
+                </div>
+                <div class="form-group">
+                    <label for="import_ignore_interval">Ignore Interval</label>
+                    <div class="row">
+                        <div class="col-xs-2">
+                            <input type="text" class="form-control" id="import_ignore_interval" name="import_ignore_interval" value="120" required>
+                        </div>
+                    </div>
+                    <p class="help-block">Enter the minimum duration (in seconds) an item must have been active for. Set to 0 to import all.</p>
+                </div>
+                % endif
+            </form>
         </div>
         <div class="modal-footer">
             <div>
@@ -55,24 +106,97 @@
     </div>
 </div>
 <script>
-    // Send database path to import script
+    $('#import_database_path_browse').click(function () {
+        $('#browse-path-type').text('Databse File');
+        $('#browse-path-modal').modal('show');
+        browsePath(null, null, '.db');
+    });
+    $('#select-browse-file').click(function () {
+        $('#browse-path-modal').modal('hide');
+        $("#import_database_path").val($('#browse-path').val());
+    });
+
+    $('#import_database_file').change(function() {
+        if ($(this)[0].files[0]) {
+            $('#import_database_file_name').val($(this)[0].files[0].name);
+        }
+    });
+
     $("#import_db").click(function() {
-        var database_path = $("#db_location").val();
-        var table_name = $("#table_name").val();
-        var import_ignore_interval = $("#import_ignore_interval").val();
+        $(this).prop('disabled', true);
+
+        var app = $("#import_app").val();
+        var database_file = $('#import_database_file')[0].files[0];
+        var database_path = $("#import_database_path").val();
+        var method = $("#import_method").val();
+        var backup = $("#import_backup_db").is(':checked');
+        var table_name = $("#import_table_name").val();
+        var ignore_interval = $("#import_ignore_interval").val();
+
+        var content_type;
+        var process_data;
+        var data;
+
+        if (database_file) {
+            content_type = false;
+            process_data = false;
+            data = new FormData();
+            data.append('app', app);
+            data.append('database_file', database_file);
+            data.append('method', method);
+            data.append('backup', backup);
+            data.append('table_name', table_name);
+            data.append('ignore_interval', ignore_interval);
+        } else {
+            content_type = 'application/x-www-form-urlencoded; charset=UTF-8';
+            process_data = true;
+            data = {
+                app: app,
+                database_path: database_path,
+                method: method,
+                backup: backup,
+                table_name: table_name,
+                ignore_interval: ignore_interval
+            }
+        }
+
+        if (database_file) {
+            $("#status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>&nbsp; Uploading database file...');
+        } else {
+            $("#status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>');
+        }
+
         $.ajax({
             url: 'import_database',
-            data: {
-                app: "${app}",
-                database_path: database_path,
-                table_name: table_name,
-                import_ignore_interval: import_ignore_interval
-            },
+            type: 'POST',
+            data: data,
             cache: false,
             async: true,
+            contentType: content_type,
+            processData: process_data,
             success: function(data) {
-                $("#status-message").html(data);
-                $("#db_location").val('')
+                var msg;
+                if (data.result === 'success') {
+                    msg = "<i class='fa fa-check'></i>&nbsp; " + data.message;
+                } else {
+                    msg = "<i class='fa fa-exclamation-triangle'></i>&nbsp; " + data.message;
+                }
+                $("#status-message").html(msg);
+                $("#import_database_file").val(null);
+                $("#import_database_file_name").val('');
+                $("#import_database_path").val('');
+            },
+            error: function (xhr) {
+                var msg = "<i class='fa fa-exclamation-triangle'></i>&nbsp; Error (" + xhr.status + "): ";
+                if (xhr.status === 413) {
+                    msg += "file is too large to upload"
+                } else {
+                    msg += 'try again'
+                }
+                $("#status-message").html(msg);
+            },
+            complete: function(xhr) {
+                $("#import_db").prop('disabled', false);
             }
         });
     });

data/interfaces/default/base.html

@@ -55,8 +55,10 @@
         newer version</a> of Tautulli is available!<br />
         You are ${plexpy.COMMITS_BEHIND} commit${'s' if plexpy.COMMITS_BEHIND > 1 else ''} behind.<br />
         % endif
-        % if plexpy.DOCKER:
+        % if plexpy.INSTALL_TYPE == 'docker':
         Update your Docker container or <a href="#" id="updateDismiss">Dismiss</a>
+        % elif plexpy.INSTALL_TYPE in ('windows', 'macos'):
+        <a href="${anon_url('https://github.com/%s/%s/releases/tag/%s' % (plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO, plexpy.LATEST_RELEASE))}" target="_blank">Download</a> and install the latest version or <a href="#" id="updateDismiss">Dismiss</a>
         % else:
         <a href="update">Update</a> or <a href="#" id="updateDismiss">Dismiss</a>
         % endif
@@ -294,9 +296,7 @@ ${next.modalIncludes()}
 <script src="${http_root}js/ipaddr.min.js"></script>
 <script src="${http_root}js/script.js${cache_param}"></script>
 <script src="${http_root}js/jquery.tripleclick.min.js"></script>
-% if _session['user_group'] == 'admin' and BROWSER_NOTIFIERS:
 <script src="${http_root}js/ajaxNotifications.js"></script>
-% endif
 <script>
     % if _session['user_group'] == 'admin':
     $('body').on('click', '#updateDismiss', function() {
@@ -330,8 +330,10 @@ ${next.modalIncludes()}
                         msg = 'A <a href="' + result.compare_url + '" target="_blank">newer version</a> of Tautulli is available!<br />' +
                             'You are '+ result.commits_behind + ' commit' + (result.commits_behind > 1 ? 's' : '') + ' behind.<br />';
                     }
-                    if (result.docker) {
+                    if (result.install_type === 'docker') {
                         msg += 'Update your Docker container or <a href="#" id="updateDismiss">Dismiss</a>';
+                    } else if (result.install_type === 'windows' || result.install_type === 'macos') {
+                        msg += '<a href="' + result.release_url + '" target="_blank">Download</a> and install the latest version or <a href="#" id="updateDismiss">Dismiss</a>'
                     } else {
                         msg += '<a href="update">Update</a> or <a href="#" id="updateDismiss">Dismiss</a>';
                     }
@@ -419,6 +421,10 @@ ${next.modalIncludes()}
         $(document).on('hidden.bs.modal', '.modal', function () {
             $('.modal:visible').length && $(document.body).addClass('modal-open');
         });
+
+        % if _session['user_group'] == 'admin' and BROWSER_NOTIFIERS:
+        check_notifications();
+        % endif
     });
 
     % if _session['user_group'] != 'admin':

data/interfaces/default/css/tautulli.css

@@ -2161,7 +2161,7 @@ div.advanced-setting {
 li.advanced-setting {
     border-left: 1px solid #cc7b19;
 }
-.docker-setting {
+.setting-message {
     color: #cc7b19;
     margin-left: 10px;
 }
@@ -2312,6 +2312,7 @@ li.advanced-setting {
     width: 140px;
     margin-left: 10px;
     margin-bottom: 10px;
+    white-space: nowrap;
 }
 .user-player-instance-playcount h3 {
     font-size: 30px;
@@ -2994,6 +2995,9 @@ a .home-platforms-list-cover-face:hover
 .accordion li .link i.fa {
     color: #999;
 }
+.accordion li .link span.toggle-left {
+    padding-right: 5px;
+}
 .accordion li .link span.toggle-right {
     float: right;
     padding-left: 10px;
@@ -3870,6 +3874,10 @@ a:hover .overlay-refresh-image:hover {
     background-color: #31afe1;
     background-image: url(../images/platforms/kodi.svg);
 }
+.platform-lg {
+    background-color: #a50034;
+    background-image: url(../images/platforms/lg.svg);
+}
 .platform-linux {
     background-color: #1793d0;
     background-image: url(../images/platforms/linux.svg);
@@ -3971,6 +3979,9 @@ a:hover .overlay-refresh-image:hover {
 .platform-kodi-rgba {
     background-color: rgba(49, 175, 225, 0.40);
 }
+.platform-lg-rgba {
+    background-color: rgba(165, 0, 52, 0.40);
+}
 .platform-linux-rgba {
     background-color: rgba(23, 147, 208, 0.40);
 }
@@ -4296,4 +4307,41 @@ a[data-tab-destination] {
 .help-block li {
     margin-top: 0;
     color: #737373;
-}
+}
+
+#browse-path-list > li > span > i.fa {
+    color: #999;
+}
+
+#tautulli-news .open .news-title,
+#tautulli-news .open .news-date,
+#tautulli-news .accordion li.open .link i.fa {
+    color: #eee;
+}
+.news-title,
+.news-date {
+    color: #999;
+    padding-left: 5px;
+}
+.news-subtitle {
+    display: block;
+    color: #aaa;
+    font-weight: bold;
+    margin-bottom: 10px;
+}
+.news-body {
+    display: block;
+    color: #aaa;
+}
+.news-body p:last-of-type {
+    margin-bottom: 0;
+}
+.news-body a {
+    display: inline !important;
+    background: none !important;
+    padding: 0 !important;
+    color: #fff;
+}
+.news-body a:hover {
+    color: #f9be03;
+}

data/interfaces/default/current_activity_instance.html

@@ -276,14 +276,17 @@ DOCUMENTATION :: END
                                 <div class="sub-heading">Subtitle</div>
                                 <div class="sub-value" id="subtitle_decision-${sk}">
                                     % if data['subtitles'] == 1:
+                                        <%
+                                            subtitle_codec = 'None' if data['stream_subtitle_codec'] and data['stream_subtitle_transient'] else data['subtitle_codec'].upper()
+                                        %>
                                         % if data['stream_subtitle_decision'] == 'transcode':
-                                        Transcode (${data['subtitle_codec'].upper()} <i class="fa fa-long-arrow-right"></i> ${data['stream_subtitle_codec'].upper()})
+                                        Transcode (${subtitle_codec} <i class="fa fa-long-arrow-right"></i> ${data['stream_subtitle_codec'].upper()})
                                         % elif data['stream_subtitle_decision'] == 'copy':
-                                        Direct Stream (${data['subtitle_codec'].upper()})
+                                        Direct Stream (${subtitle_codec})
                                         % elif data['stream_subtitle_decision'] == 'burn':
-                                        Burn (${data['subtitle_codec'].upper()})
+                                        Burn (${subtitle_codec})
                                         % else:
-                                        Direct Play (${data['subtitle_codec'].upper() if data['synced_version'] else data['stream_subtitle_codec'].upper()})
+                                        Direct Play (${subtitle_codec if data['synced_version'] else data['stream_subtitle_codec'].upper()})
                                         % endif
                                     % else:
                                     None

data/interfaces/default/history.html

@@ -8,6 +8,13 @@
 
 <%def name="body()">
 <div class='container-fluid'>
+    % if config['database_is_importing']:
+    <div style="text-align: center; margin-top: 20px;">
+        <i class="fa fa-refresh fa-spin"></i>&nbsp; Tautulli is importing history from another database. This could take a few minutes depending on the size of your database.
+        <br />
+        You may leave this page and check back later.
+    </div>
+    % endif
     <div class='table-card-header'>
         <div class="header-bar">
             <span><i class="fa fa-history"></i> History</span>
@@ -193,9 +200,9 @@
                             success: function (data) {
                                 var msg = "History deleted";
                                 showMsg(msg, false, true, 2000);
+                                history_table.draw();
                             }
                         });
-                        history_table.draw();
                     });
                 }
 

data/interfaces/default/images/platforms/lg.svg

@@ -0,0 +1,7 @@
+<!-- Generated by IcoMoon.io -->
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="64" height="64" viewBox="0 0 64 64">
+<title>lg</title>
+<path fill="#fff" d="M30.203 31.797c0 8.176-6.654 14.832-14.835 14.82-7.927-0.011-14.818-6.28-14.812-14.838 0.005-8.282 6.541-14.82 14.841-14.803 8.618 0.017 14.807 6.969 14.806 14.822zM26.577 32.388c-0.087 4.433-3.485 9.518-9.37 10.487-6.122 1.008-11.584-2.989-12.814-8.656-0.632-2.912-0.221-5.696 1.362-8.228 2.347-3.754 5.815-5.502 10.222-5.453 0-0.387 0-0.761 0-1.134-4.114-0.281-9.226 1.824-11.763 6.923-2.454 4.932-1.296 10.953 2.811 14.672 4.153 3.762 10.224 4.309 14.953 1.326 2.328-1.468 3.999-3.496 4.997-6.067 0.628-1.617 0.882-3.296 0.813-5.032-2.967 0-5.909 0-8.864 0 0 0.39 0 0.768 0 1.162 2.558-0 5.097-0 7.652-0zM15.991 37.112c0-0.129 0-0.221 0-0.313 0-3.731 0-7.463 0-11.194 0-0.060-0.004-0.119 0-0.179 0.009-0.118-0.038-0.166-0.16-0.163-0.278 0.006-0.556 0.012-0.833-0.002-0.178-0.008-0.237 0.042-0.237 0.23 0.005 4.194 0.005 8.389-0 12.583-0 0.198 0.065 0.239 0.249 0.237 1.224-0.007 2.448-0.004 3.672-0.004 0.072 0 0.143 0 0.244 0 0-0.343-0.008-0.665 0.003-0.987 0.006-0.166-0.050-0.214-0.214-0.212-0.82 0.007-1.641 0.003-2.461 0.003-0.078 0-0.155 0-0.263 0zM12.434 27.068c0.003-0.987-0.799-1.798-1.785-1.805s-1.799 0.796-1.805 1.782c-0.006 0.985 0.799 1.8 1.783 1.805 0.985 0.004 1.804-0.803 1.807-1.783z"></path>
+<path fill="#fff" d="M63.467 30.606c0 2.864 0 5.707 0 8.571-1.242 0-2.479 0-3.742 0 0-0.468 0-0.933 0-1.433-0.203 0.226-0.366 0.432-0.553 0.612-0.683 0.656-1.518 1-2.441 1.136-1.187 0.174-2.348 0.075-3.462-0.4-1.234-0.526-2.145-1.407-2.8-2.565-0.599-1.058-0.906-2.207-1.035-3.409-0.148-1.367-0.103-2.723 0.28-4.051 0.797-2.764 2.635-4.391 5.453-4.899 1.534-0.277 3.058-0.208 4.54 0.311 1.243 0.436 2.298 1.139 3.011 2.276 0.431 0.688 0.584 1.467 0.687 2.258 0.013 0.097 0.028 0.195 0.046 0.318-0.064 0.003-0.126 0.010-0.188 0.010-1.389 0.001-2.779-0.002-4.169 0.003-0.151 0.001-0.215-0.034-0.245-0.197-0.229-1.234-1.281-1.773-2.308-1.679-1.182 0.108-1.823 0.859-2.22 1.888-0.211 0.547-0.315 1.12-0.352 1.703-0.066 1.061-0.039 2.117 0.31 3.138 0.211 0.618 0.523 1.173 1.050 1.579 1.371 1.055 3.326 0.436 3.877-1.228 0.090-0.274 0.157-0.557 0.246-0.875-0.112 0-0.182 0-0.251 0-0.794 0-1.588-0.005-2.382 0.004-0.168 0.002-0.213-0.053-0.212-0.214 0.006-0.887 0.005-1.774 0.001-2.66-0.001-0.139 0.019-0.215 0.192-0.215 2.17 0.006 4.341 0.004 6.511 0.004 0.045 0 0.090 0.008 0.157 0.013z"></path>
+<path fill="#fff" d="M48.501 35.522c0 1.233 0 2.44 0 3.661-3.613 0-7.216 0-10.834 0 0-4.923 0-9.841 0-14.77 1.44 0 2.872 0 4.331 0 0 0.092 0 0.175 0 0.259 0 3.526 0 7.053 0 10.579 0 0.271 0 0.271 0.267 0.271 1.985 0 3.97 0 5.954 0 0.086 0 0.171 0 0.281 0z"></path>
+</svg>

data/interfaces/default/index.html

@@ -493,14 +493,15 @@
 
                         var subtitle_decision = 'None';
                         if (['movie', 'episode', 'clip'].indexOf(s.media_type) > -1 && s.subtitles === 1) {
+                            var subtitle_codec = (s.stream_subtitle_codec && s.stream_subtitle_transient) ? 'None' : s.subtitle_codec.toUpperCase();
                             if (s.stream_subtitle_decision === 'transcode') {
-                                subtitle_decision = 'Transcode (' + s.subtitle_codec.toUpperCase() + ' <i class="fa fa-long-arrow-right"></i> ' + s.stream_subtitle_codec.toUpperCase() + ')';
+                                subtitle_decision = 'Transcode (' + subtitle_codec + ' <i class="fa fa-long-arrow-right"></i> ' + s.stream_subtitle_codec.toUpperCase() + ')';
                             } else if (s.stream_subtitle_decision === 'copy') {
-                                subtitle_decision = 'Direct Stream (' + s.subtitle_codec.toUpperCase() + ')';
+                                subtitle_decision = 'Direct Stream (' + subtitle_codec + ')';
                             } else if (s.stream_subtitle_decision === 'burn') {
-                                subtitle_decision = 'Burn (' + s.subtitle_codec.toUpperCase() + ')';
+                                subtitle_decision = 'Burn (' + subtitle_codec + ')';
                             } else {
-                                subtitle_decision = 'Direct Play (' + ((s.synced_version === '1') ? s.subtitle_codec.toUpperCase() : s.stream_subtitle_codec.toUpperCase()) + ')';
+                                subtitle_decision = 'Direct Play (' + ((s.synced_version === '1') ? subtitle_codec : s.stream_subtitle_codec.toUpperCase()) + ')';
                             }
                         }
                         $('#subtitle_decision-' + key).html(subtitle_decision);

data/interfaces/default/info.html

@@ -45,14 +45,14 @@ DOCUMENTATION :: END
 
     # Get audio codec file
     def af(codec):
-        for pattern, file_type in MEDIA_FLAGS_AUDIO.iteritems():
+        for pattern, file_type in MEDIA_FLAGS_AUDIO.items():
             if re.match(pattern, codec):
                 return file_type
         return codec
 
     # Get video codec file
     def vf(codec):
-        for pattern, file_type in MEDIA_FLAGS_VIDEO.iteritems():
+        for pattern, file_type in MEDIA_FLAGS_VIDEO.items():
             if re.match(pattern, codec):
                 return file_type
         return codec
@@ -729,9 +729,9 @@ DOCUMENTATION :: END
                             success: function (data) {
                                 var msg = "History deleted";
                                 showMsg(msg, false, true, 2000);
+                                history_table.draw();
                             }
                         });
-                        history_table.draw();
                     });
                 }
 

data/interfaces/default/ip_address_modal.html

@@ -24,7 +24,6 @@
             <div id="ip_error" class="col-sm-12 text-muted"></div>
             <div class="col-sm-6">
                 <ul class="list-unstyled">
-                    <li>Continent: <strong><span id="continent"></span></strong></li>
                     <li>Country: <strong><span id="country"></span></strong></li>
                     <li>Region: <strong><span id="region"></span></strong></li>
                     <li>City: <strong><span id="city"></span></strong></li>
@@ -36,7 +35,6 @@
                     <li>Timezone: <strong><span id="timezone"></span></strong></li>
                     <li>Latitude: <strong><span id="latitude"></span></strong></li>
                     <li>Longitude: <strong><span id="longitude"></span></strong></li>
-                    <li>Accuracy Radius: <strong><span id="accuracy"></span></strong></li>
                 </ul>
             </div>
             <div class="col-sm-12">
@@ -61,8 +59,6 @@
             </div>
         </div>
         <div class="modal-footer">
-            <% from plexpy.helpers import anon_url %>
-            <span class="text-muted">GeoLite2 data created by <a href="${anon_url('http://www.maxmind.com')}" target="_blank">MaxMind</a>.</span>
         </div>
     </div>
 </div>
@@ -82,11 +78,11 @@
             error: function () {
                 $('#ip_error').html('<i class="fa fa-exclamation-circle"></i> Internal request failed.').show();
             },
-            success: function (data) {
-                if ('error' in data) {
-                    $('#ip_error').html('<i class="fa fa-exclamation-circle"></i> ' + data.error).show();
+            success: function (result) {
+                if (result.result === 'error') {
+                    $('#ip_error').html('<i class="fa fa-exclamation-circle"></i> ' + result.message).show();
                 } else {
-                    $('#continent').html(data.continent);
+                    var data = result.data;
                     $('#country').html(data.country);
                     $('#region').html(data.region);
                     $('#city').html(data.city);
@@ -94,7 +90,6 @@
                     $('#timezone').html(data.timezone);
                     $('#latitude').html(data.latitude);
                     $('#longitude').html(data.longitude);
-                    $('#accuracy').html(data.accuracy + ' km');
                 }
             }
         });

data/interfaces/default/js/ajaxNotifications.js

@@ -36,7 +36,3 @@ function check_notifications() {
         check_notifications();
     }, 5000);
 }
-
-$(document).ready(function () {
-    check_notifications();
-});

data/interfaces/default/js/script.js

@@ -237,6 +237,27 @@ function doAjaxCall(url, elem, reload, form, showMsg, callback) {
     });
 }
 
+getBrowsePath = function (key, path, filter_ext) {
+    var deferred = $.Deferred();
+
+    $.ajax({
+        url: 'browse_path',
+        type: 'GET',
+        data: {
+            key: key,
+            path: path,
+            filter_ext: filter_ext
+        },
+        success: function(data) {
+            deferred.resolve(data);
+        },
+        error: function() {
+            deferred.reject();
+        }
+    });
+    return deferred;
+};
+
 function doSimpleAjaxCall(url) {
     $.ajax(url);
 }

data/interfaces/default/library.html

@@ -251,9 +251,9 @@ DOCUMENTATION :: END
                                     % else:
                                     <div id="get_file_sizes_message" style="text-align: center; margin-top: 20px; display: none;">
                                     % endif
-                                        <i class="fa fa-refresh fa-spin"></i> Tautulli is calculating the file sizes for the library's media info. This could take a few minutes depending on the size of your library.
+                                        <i class="fa fa-refresh fa-spin"></i>&nbsp; Tautulli is calculating the file sizes for the library's media info. This could take a few minutes depending on the size of your library.
                                         <br />
-                                        You may leave this page and come back later.
+                                        You may leave this page and check back later.
                                     </div>
                                     <div class='table-card-header'>
                                         <div class="header-bar">
@@ -493,9 +493,9 @@ DOCUMENTATION :: END
                         success: function (data) {
                             var msg = "History deleted";
                             showMsg(msg, false, true, 2000);
+                            history_table.draw();
                         }
                     });
-                    history_table.draw();
                 });
             }
 

data/interfaces/default/login.html

@@ -24,7 +24,7 @@
 
     <!-- ICONS -->
     <!-- Android -->
-    <link rel="manifest" href="${http_root}images/favicon/manifest.json?v=2.0.5" crossorigin="use-credentials>
+    <link rel="manifest" href="${http_root}images/favicon/manifest.json?v=2.0.5" crossorigin="use-credentials">
     <meta name="theme-color" content="#282a2d">
     <!-- Apple -->
     <link rel="apple-touch-icon" sizes="180x180" href="${http_root}images/favicon/apple-touch-icon.png?v=2.0.5">

data/interfaces/default/mobile_devices_table.html

@@ -13,7 +13,11 @@ DOCUMENTATION :: END
     % for device in sorted(devices_list, key=lambda k: k['device_name']):
     <li class="mobile-device pointer" data-id="${device['id']}" data-name="${device['device_name']}">
         <span>
-            <span class="toggle-left"><i class="fa fa-lg fa-fw fa-mobile"></i></span>
+            % if device['official']:
+                <span class="toggle-left"><i class="fa fa-lg fa-fw fa-mobile"></i></span>
+            % else:
+                <span class="toggle-left officail-tooltip" data-toggle="tooltip" data-placement="top" title="Unofficial or Unknown App"><i class="fa fa-lg fa-fw fa-exclamation-triangle"></i></span>
+            % endif
             ${device['friendly_name'] or device['device_name']} &nbsp;<span class="friendly_name">(${device['id']})</span>
             <span class="toggle-right"><i class="fa fa-lg fa-fw fa-cog"></i></span>
             <span class="toggle-right friendly_name" id="device-last_seen-${device['id']}">
@@ -138,4 +142,6 @@ DOCUMENTATION :: END
         }
         verifiedDevice = true;
     })
+
+    $('.officail-tooltip').tooltip();
 </script>

data/interfaces/default/newsletter_config.html

@@ -123,7 +123,7 @@
                                         <div class="row">
                                             <div class="col-md-12">
                                                 <select class="form-control" id="${item['name']}" name="${item['name']}">
-                                                    % for key, value in sorted(item['select_options'].iteritems()):
+                                                    % for key, value in sorted(item['select_options'].items()):
                                                     % if key == item['value']:
                                                     <option value="${key}" selected>${value}</option>
                                                     % else:
@@ -144,7 +144,7 @@
                                                     <option value="select-all">Select All</option>
                                                     <option value="remove-all">Remove All</option>
                                                     % if isinstance(item['select_options'], dict):
-                                                    % for section, options in item['select_options'].iteritems():
+                                                    % for section, options in item['select_options'].items():
                                                     <optgroup label="${section}">
                                                         % for option in sorted(options, key=lambda x: x['text'].lower()):
                                                             <option value="${option['value']}">${option['text']}</option>
@@ -325,7 +325,7 @@
                                             <div class="row">
                                                 <div class="col-md-12">
                                                     <select class="form-control" id="${item['name']}" name="${item['name']}">
-                                                        % for key, value in sorted(item['select_options'].iteritems()):
+                                                        % for key, value in sorted(item['select_options'].items()):
                                                         % if key == item['value']:
                                                         <option value="${key}" selected>${value}</option>
                                                         % else:
@@ -346,7 +346,7 @@
                                                         <option value="select-all">Select All</option>
                                                         <option value="remove-all">Remove All</option>
                                                         % if isinstance(item['select_options'], dict):
-                                                        % for section, options in item['select_options'].iteritems():
+                                                        % for section, options in item['select_options'].items():
                                                         <optgroup label="${section}">
                                                             % for option in sorted(options, key=lambda x: x['text'].lower()):
                                                                 <option value="${option['value']}">${option['text']}</option>

data/interfaces/default/newsletter_preview.html

@@ -1,5 +1,5 @@
 <%
-    import urllib
+    from six.moves.urllib.parse import urlencode
 %>
 <!doctype html>
 
@@ -35,7 +35,7 @@
 <script>
 $(document).ready(function () {
     var frame = $('<iframe></iframe>', {
-        src: 'real_newsletter?${urllib.urlencode(kwargs) | n}',
+        src: 'real_newsletter?${urlencode(kwargs) | n}',
         frameborder: '0',
         style: 'display: none; height: 100vh; width: 100vw;'
     });

data/interfaces/default/newsletters_table.html

@@ -9,7 +9,7 @@ Version:            0.1
 DOCUMENTATION :: END
 </%doc>
 
-<% from plexpy.newsletter_handler import NEWSLETTER_SCHED %>
+<% from plexpy import newsletter_handler %>
 <ul class="stacked-configs list-unstyled">
     % for newsletter in sorted(newsletters_list, key=lambda k: (k['agent_label'], k['friendly_name'], k['id'])):
     <li class="newsletter-agent pointer" data-id="${newsletter['id']}">
@@ -22,8 +22,8 @@ DOCUMENTATION :: END
             % endif
             <span class="toggle-right"><i class="fa fa-lg fa-fw fa-cog"></i></span>
             <span class="toggle-right friendly_name" id="newsletter-next_run-${newsletter['id']}">
-                % if NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])):
-                <% job = NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])) %>
+                % if newsletter_handler.NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])):
+                <% job = newsletter_handler.NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])) %>
                 <script>
                     $("#newsletter-next_run-${newsletter['id']}").text(moment("${job.next_run_time}", "YYYY-MM-DD HH:mm:ssZ").fromNow())
                 </script>

data/interfaces/default/notifier_config.html

@@ -1,9 +1,9 @@
 % if notifier:
-<%!
+<%
     import json
     from plexpy import notifiers, users
     from plexpy.helpers import checked
-    available_notification_actions = notifiers.available_notification_actions()
+    available_notification_actions = notifiers.available_notification_actions(agent_id=notifier['agent_id'])
 
     user_emails = [{'user': u['friendly_name'] or u['username'], 'email': u['email']} for u in users.Users().get_users() if u['email']]
     sorted(user_emails, key=lambda u: u['user'])
@@ -25,7 +25,7 @@
                         <li role="presentation"><a href="#tabs-notify_text" aria-controls="tabs-notify_text" role="tab" data-toggle="tab">Arguments</a></li>
                         % elif notifier['agent_name'] == 'webhook':
                         <li role="presentation"><a href="#tabs-notify_text" aria-controls="tabs-notify_text" role="tab" data-toggle="tab">Data</a></li>
-                        % else:
+                        % elif notifier['agent_name'] != 'plexmobileapp':
                         <li role="presentation"><a href="#tabs-notify_text" aria-controls="tabs-notify_text" role="tab" data-toggle="tab">Text</a></li>
                         % endif
                         <li role="presentation"><a href="#tabs-test_notifications" aria-controls="tabs-test_notifications" role="tab" data-toggle="tab">Test Notifications</a></li>
@@ -88,7 +88,7 @@
                                         <div class="row">
                                             <div class="col-md-12">
                                                 <select class="form-control" id="${item['name']}" name="${item['name']}">
-                                                    % for key, value in sorted(item['select_options'].iteritems()):
+                                                    % for key, value in sorted(item['select_options'].items()):
                                                     % if key == item['value']:
                                                     <option value="${key}" selected>${value}</option>
                                                     % else:
@@ -109,7 +109,7 @@
                                                     <option value="select-all">Select All</option>
                                                     <option value="remove-all">Remove All</option>
                                                     % if isinstance(item['select_options'], dict):
-                                                    % for section, options in item['select_options'].iteritems():
+                                                    % for section, options in item['select_options'].items():
                                                     <optgroup label="${section}">
                                                         % for option in sorted(options, key=lambda x: x['text'].lower()):
                                                             <option value="${option['value']}">${option['text']}</option>
@@ -211,7 +211,7 @@
                                 % for action in available_notification_actions:
                                 <li>
                                     <div class="link">
-                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>&nbsp;
+                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>
                                         ${action['label']}
                                         <span class="toggle-right"><i class="fa fa-chevron-down"></i></span>
                                     </div>
@@ -237,7 +237,7 @@
                                 % for action in available_notification_actions:
                                 <li>
                                     <div class="link">
-                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>&nbsp;
+                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>
                                         ${action['label']}
                                         <span class="toggle-right"><i class="fa fa-chevron-down"></i></span>
                                     </div>
@@ -268,7 +268,7 @@
                                 % for action in available_notification_actions:
                                 <li>
                                     <div class="link">
-                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>&nbsp;
+                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>
                                         ${action['label']}
                                         <span class="toggle-right"><i class="fa fa-chevron-down"></i></span>
                                     </div>
@@ -313,7 +313,7 @@
                                 <div class="row">
                                     <div class="col-md-12">
                                         <select class="form-control" id="test_script" name="test_script">
-                                            % for key, value in sorted(notifier['config_options'][2]['select_options'].iteritems()):
+                                            % for key, value in sorted(notifier['config_options'][2]['select_options'].items()):
                                             <option value="${key}">${value}</option>
                                             % endfor
                                         </select>
@@ -684,6 +684,15 @@
         pushoverPriority();
     });
 
+    % elif notifier['agent_name'] == 'plexmobileapp':
+    var $plexmobileapp_user_ids = $('#plexmobileapp_user_ids').selectize({
+        plugins: ['remove_button'],
+        maxItems: null,
+        create: true
+    });
+    var plexmobileapp_user_ids = $plexmobileapp_user_ids[0].selectize;
+    plexmobileapp_user_ids.setValue(${json.dumps(next((c['value'] for c in notifier['config_options'] if c['name'] == 'plexmobileapp_user_ids'), [])) | n});
+
     % endif
 
     function validateLogic() {
@@ -844,10 +853,8 @@
                 PNotify.prototype.options.hide = true;
                 PNotify.prototype.options.delay = $('#browser_auto_hide_delay').val() * 1000;
             }
-            var notification = new PNotify({
-                title: $('#test_subject').val(),
-                text: $('#test_body').val()
-            });
+            displayPNotify($('#test_subject').val(), $('#test_body').val());
+            showMsg('<i class="fa fa-check"></i> Notification sent.', false, true, 5000);
         }
     }
 

data/interfaces/default/settings.html

@@ -8,7 +8,7 @@
     from plexpy.helpers import anon_url, checked
 
     docker_setting = 'disabled' if plexpy.DOCKER else ''
-    docker_msg = '<span class="docker-setting small">(Controlled by Docker Container)</span>' if plexpy.DOCKER else ''
+    docker_msg = '<span class="setting-message small">(Controlled by Docker Container)</span>' if plexpy.DOCKER else ''
 
     available_notification_agents = sorted(notifiers.available_notification_agents(), key=lambda k: k['label'].lower())
     available_newsletter_agents = sorted(newsletters.available_newsletter_agents(), key=lambda k: k['label'].lower())
@@ -71,6 +71,13 @@
                             <h3>Version ${common.RELEASE} <small><a id="changelog-modal-link" href="#"><i class="fa fa-info-circle"></i> Changelog</a></small></h3>
                         </div>
                         % endif
+                        <div class="padded-header">
+                            <h3>Tautulli News</h3>
+                        </div>
+                        <div id="tautulli-news">
+                            <div class='text-muted'><i class="fa fa-refresh fa-spin"></i> Loading news...</div>
+                            <br>
+                        </div>
                         <div class="padded-header">
                             <h3>Tautulli Configuration</h3>
                         </div>
@@ -215,12 +222,14 @@
                             <p class="help-block">Check for Tautulli updates periodically.</p>
                         </div>
                         <div id="git_update_options">
+                            % if not plexpy.FROZEN:
                             <div class="checkbox">
                                 <label>
                                     <input type="checkbox" id="plexpy_auto_update" name="plexpy_auto_update" value="1" ${config['plexpy_auto_update']} ${docker_setting}> Update Automatically ${docker_msg | n}
                                 </label>
                                 <p class="help-block">Update Tautulli automatically if an update is available.</p>
                             </div>
+                            % endif
                             <div class="form-group advanced-setting">
                                 <label for="git_token">GitHub API Token</label>
                                 <div class="row">
@@ -448,12 +457,27 @@
                         </div>
 
                         <p class="help-block">Note: Web interface changes require a restart.</p>
-                        % if os.name == 'nt':
+                        % if common.PLATFORM in ('Windows', 'Darwin'):
+                        <%
+                            tray = {'Windows': 'System Tray', 'Darwin': 'Menu Bar'}
+                            tray_disabled = tray_disabled_msg = ''
+                            if common.PLATFORM == 'Darwin':
+                                from plexpy.macos import HAS_PYOBJC
+                                if not HAS_PYOBJC:
+                                    tray_disabled = 'disabled'
+                                    tray_disabled_msg = '<span class="setting-message small">(Missing pyobjc module)</span>'
+                        %>
                         <div class="checkbox">
                             <label>
-                                <input type="checkbox" class="http-settings" name="win_sys_tray" id="win_sys_tray" value="1" ${config['win_sys_tray']}> Enable System Tray Icon
+                                <input type="checkbox" class="http-settings" name="sys_tray_icon" id="sys_tray_icon" value="1" ${config['sys_tray_icon']} ${tray_disabled}> Enable ${tray[common.PLATFORM]} Icon ${tray_disabled_msg | n}
                             </label>
-                            <p class="help-block">Show Tautulli shortcut in the system tray.</p>
+                            <p class="help-block">Show Tautulli shortcut in the ${tray[common.PLATFORM].lower()}.</p>
+                        </div>
+                        <div class="checkbox">
+                            <label>
+                                <input type="checkbox" name="launch_startup" id="launch_startup" value="1" ${config['launch_startup']}> Launch at System Startup
+                            </label>
+                            <p class="help-block">Start Tautulli automatically after Login.</p>
                         </div>
                         % endif
                         <div class="checkbox">
@@ -850,6 +874,28 @@
                             <span id="remoteAccessCheck" class="settings-warning"></span>
                             <p class="help-block">Enable to have Tautulli check if remote access to the Plex Media Server goes down.</p>
                         </div>
+                        <div id="monitor_remote_access_options">
+                            <div class="form-group advanced-setting">
+                                <label for="remote_access_ping_interval">Remote Access Ping Interval</label>
+                                <div class="row">
+                                    <div class="col-md-2">
+                                        <input type="text" class="form-control" data-parsley-type="integer" id="remote_access_ping_interval" name="remote_access_ping_interval" value="${config['remote_access_ping_interval']}" size="5" data-parsley-min="60" data-parsley-trigger="change" data-parsley-errors-container="#remote_access_ping_interval_error" required>
+                                    </div>
+                                    <div id="remote_access_ping_interval_error" class="alert alert-danger settings-alert" role="alert"></div>
+                                </div>
+                                <p class="help-block">The interval (in seconds) Tautulli will ping the Plex Media Server for the remote access status. Minimum 60.</p>
+                            </div>
+                            <div class="form-group advanced-setting">
+                                <label for="remote_access_ping_threshold">Remote Access Ping Threshold</label>
+                                <div class="row">
+                                    <div class="col-md-2">
+                                        <input type="text" class="form-control" data-parsley-type="integer" id="remote_access_ping_threshold" name="remote_access_ping_threshold" value="${config['remote_access_ping_threshold']}" size="5" data-parsley-min="1" data-parsley-trigger="change" data-parsley-errors-container="#remote_access_ping_threshold_error" required>
+                                    </div>
+                                    <div id="remote_access_ping_threshold_error" class="alert alert-danger settings-alert" role="alert"></div>
+                                </div>
+                                <p class="help-block">The number of consecutive remote access status failures to consider remote access as down. Minimum 1.</p>
+                            </div>
+                        </div>
 
                         <div class="form-group advanced-setting">
                             <label for="refresh_users_interval">Users List Refresh Interval</label>
@@ -941,7 +987,7 @@
                                 </div>
                                 <div id="buffer_wait_error" class="alert alert-danger settings-alert" role="alert"></div>
                             </div>
-                            <p class="help-block">The value (in seconds) Tautulli should wait before triggering the next buffer warning. 0 to always trigger.</p>
+                            <p class="help-block">The value (in seconds) Tautulli should wait before triggering the next buffer warning. Set to 0 to always trigger.</p>
                         </div>
                         <div class="checkbox advanced-setting">
                             <label>
@@ -965,6 +1011,20 @@
                             </div>
                             <p class="help-block">The number of concurrent streams by a single user for Tautulli to trigger a notification. Minimum 2.</p>
                         </div>
+                        <div class="form-group advanced-setting">
+                            <label for="notify_concurrent_threshold">Continued Session Threshold</label>
+                            <div class="row">
+                                <div class="col-md-2">
+                                    <input type="text" class="form-control" data-parsley-type="integer" id="notify_continued_session_threshold" name="notify_continued_session_threshold" value="${config['notify_continued_session_threshold']}" data-parsley-min="0" data-parsley-trigger="change" data-parsley-errors-container="#notify_continued_session_threshold_error" required>
+                                </div>
+                                <div id="notify_continued_session_threshold_error" class="alert alert-danger settings-alert" role="alert"></div>
+                            </div>
+                            <p class="help-block">
+                                The number of seconds between stopping and starting a new stream to be considered as a continued session. Set to 0 to consider all streams as new sessions.
+                                <br>
+                                Note: The threshold is only used by the "Initial Stream" notification parameter to determine if a stream is the first stream of a continuous streaming session.
+                            </p>
+                        </div>
 
                         <div class="padded-header">
                             <h3>Recently Added Notifications</h3>
@@ -996,7 +1056,7 @@
                                 </div>
                                 <div id="notify_recently_added_delay_error" class="alert alert-danger settings-alert" role="alert"></div>
                             </div>
-                            <p class="help-block">Set the delay (in seconds) to wait for consecutive recently added items to group together and to allow metadata to be processed before sending the notification. Minimum 60 seconds.</p>
+                            <p class="help-block">Set the delay (in seconds) to wait for consecutive recently added items to group together and to allow metadata to be processed before sending the recently added notification. Minimum 60 seconds, default 300.</p>
                         </div>
                         <div class="form-group advanced-setting">
                             <label>Flush Recently Added</label>
@@ -1254,7 +1314,7 @@
                             <p class="help-block">Enable to lookup links to MusicBrainz for music when available.</p>
                         </div>
                         <div class="form-group">
-                            <label for="maxmind_license_key">Delete Lookup Info</label>
+                            <label for="delete_lookup_info">Delete Lookup Info</label>
                             <p class="help-block">Delete all cached metadata lookup info in Tautulli.</p>
                             <div class="row">
                                 <div class="col-md-9">
@@ -1267,54 +1327,6 @@
                             </div>
                         </div>
 
-                        <div class="padded-header">
-                            <h3>Geolocation Database</h3>
-                        </div>
-
-                        <p class="help-block">The GeoLite2 database is used to geolocate IP addresses.</p>
-                        <p class="help-block">
-                            Please see the <a target='_blank' href='${anon_url('https://github.com/%s/%s-Wiki/wiki/3rd-Party-APIs-Guide' % (plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO))}'>3rd Party APIs Guide</a> for instructions on setting up MaxMind.<br>
-                        </p>
-                        <div class="form-group">
-                            <label for="maxmind_license_key">MaxMind License Key</label>
-                            <div class="row">
-                                <div class="col-md-6">
-                                    <input type="text" class="form-control" id="maxmind_license_key" name="maxmind_license_key" value="${config['maxmind_license_key']}" data-parsley-trigger="change">
-                                </div>
-                            </div>
-                            <p class="help-block">
-                                Enter your MaxMind License Key to install the GeoLite2 database.
-                            </p>
-                        </div>
-                        <div class="form-group">
-                            <label for="geoip_db">GeoLite2 Database File</label> ${docker_msg | n}
-                            <div class="row">
-                                <div class="col-md-9">
-                                    <div class="input-group">
-                                        <input type="text" class="form-control" id="geoip_db" name="geoip_db" value="${config['geoip_db']}" ${docker_setting} data-parsley-trigger="change" data-parsley-pattern=".+\.mmdb$" data-parsley-errors-container="#geoip_db_error"  data-parsley-error-message="Must end with '.mmdb'">
-                                        <span class="input-group-btn">
-                                            <button class="btn btn-form" type="button" id="install_geoip_db">${'Update' if config["geoip_db_installed"] else 'Install'}</button>
-                                            <button class="btn btn-form" type="button" id="uninstall_geoip_db" ${'disabled' if not config['geoip_db_installed'] else ''}>Uninstall</button>
-                                        </span>
-                                    </div>
-                                </div>
-                                <div id="geoip_db_error" class="alert alert-danger settings-alert" role="alert"></div>
-                            </div>
-                            <p class="help-block">
-                                Leave blank to install in the default location. GeoLite2 database last updated <strong><span id="geoip_db_updated">never</span></strong>.
-                            </p>
-                        </div>
-                        <div class="form-group advanced-setting">
-                            <label for="geoip_db_update_days">GeoLite2 Database Update Interval</label>
-                            <div class="row">
-                                <div class="col-md-2">
-                                    <input type="text" class="form-control" data-parsley-type="integer" id="geoip_db_update_days" name="geoip_db_update_days" value="${config['geoip_db_update_days']}" size="5" data-parsley-range="[7, 30]" data-parsley-trigger="change" data-parsley-errors-container="#geoip_db_update_days_error" required>
-                                </div>
-                                <div id="geoip_db_update_days_error" class="alert alert-danger settings-alert" role="alert"></div>
-                            </div>
-                            <p class="help-block">The interval (in days) Tautulli will automatically update the GeoLite2 database. Minimum 7, maximum 30, default 30.</p>
-                        </div>
-
                         <p><input type="button" class="btn btn-bright save-button" value="Save" data-success="Changes saved successfully"></p>
 
                     </div>
@@ -1325,8 +1337,9 @@
                             <h3>Database Import</h3>
                         </div>
 
-                        <p class="help-block">Click a button below to import an existing database from another app.</p>
+                        <p class="help-block">Click a button below to import an existing database from the selected app.</p>
                         <div class="btn-group">
+                            <button class="btn btn-form toggle-app-import-modal" type="button" data-target="#app-import-modal" data-toggle="modal" data-app="tautulli">Tautulli</button>
                             <button class="btn btn-form toggle-app-import-modal" type="button" data-target="#app-import-modal" data-toggle="modal" data-app="plexwatch">PlexWatch</button>
                             <button class="btn btn-form toggle-app-import-modal" type="button" data-target="#app-import-modal" data-toggle="modal" data-app="plexivity">Plexivity</button>
                         </div>
@@ -1422,6 +1435,7 @@
                             <label>Registered Devices</label>
                             <p class="help-block">Register a new device using a QR code, or configure an existing device by clicking the settings icon on the right.</p>
                             <p id="app_api_msg" style="color: #eb8600;">Warning: The API must be enabled under <a data-tab-destination="web_interface" data-target="api_enabled">Web Interface</a> to use the app.</p>
+                            <br />
                             <div class="row">
                                 <div id="plexpy-mobile-devices-table" class="col-md-12">
                                     <div class='text-muted'><i class="fa fa-refresh fa-spin"></i> Loading registered devices...</div>
@@ -1857,7 +1871,10 @@ Rating: {rating}/10   --> Rating: /10
                 <label>Instructions</label>
                 <p class="help-block">
                     Scan the QR code below with the Tautulli Android app to automatically register it with the server (make sure the Tautulli Address below is correct)
-                    or manually enter the connection info and device token into the app settings.
+                    or manually enter the connection info and device token into the app settings. This window will automatically close once device registration is successful.
+                </p>
+                <p class="help-block">
+                    Note: OneSignal.com must not be blocked (e.g. in Pi-hole) for device registration.
                 </p>
                 <label>QR Code</label>
                 <pre id="api_qr_code" style="text-align: center"></pre>
@@ -1885,6 +1902,38 @@ Rating: {rating}/10   --> Rating: /10
     </div>
 </div>
 <div id="mobile-device-config-modal" class="modal fade wide" tabindex="-1" role="dialog" aria-labelledby="mobile-device-config-modal"></div>
+<div id="browse-path-modal" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="browse-path-modal">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-hidden="true"><i class="fa fa-remove"></i></button>
+                <h4 class="modal-title">File Browser</h4>
+            </div>
+            <div class="modal-body">
+                <div class="row">
+                    <div class="col-md-12">
+                        <div class="form-group">
+                            <label for="browse-path">Select a <span id="browse-path-type"></span> Below</label>
+                            <div class="row">
+                                <div class="col-md-12">
+                                    <input type="text" class="form-control" id="browse-path" name="browse-path" value="" size="30" disabled>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                    <div class="col-md-12" style="height: 400px; overflow: auto;">
+                        <ul id="browse-path-list" class="stacked-configs list-unstyled">
+                        </ul>
+                    </div>
+                </div>
+            </div>
+            <div class="modal-footer">
+                <span id="browse-path-status-message" style="padding-right: 25px;"></span>
+                <input type="button" id="select-browse-file" class="btn btn-bright" value="Select">
+            </div>
+        </div>
+    </div>
+</div>
 </%def>
 
 <%def name="javascriptIncludes()">
@@ -1991,6 +2040,38 @@ Rating: {rating}/10   --> Rating: /10
         });
     }
 
+    function browsePath(key, path, filter_ext) {
+        $("#browse-path-status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>');
+        getBrowsePath(key, path, filter_ext).then(function (data) {
+            if (data.result === 'error') {
+                $("#browse-path-status-message").html("<i class='fa fa-exclamation-triangle'></i> " + data.message);
+            } else {
+                $("#browse-path-status-message").html("");
+
+                $('#browse-path').val(data.path);
+                var browse_list = $('#browse-path-list');
+                browse_list.parent().animate({ scrollTop: 0 }, 0);
+                browse_list.empty();
+
+                $.each(data.data, function(i, item) {
+                    var browse_item = $('<li/>')
+                        .html("<span><i class='fa fa-fw fa-" + item.icon + "'></i>&nbsp; " + item.title + "</span>")
+                        .addClass(item.type + ' pointer')
+                        .data('key', item.key)
+                        .data('path', item.path)
+                        .appendTo(browse_list)
+                });
+
+                $('#browse-path-list li').click(function (){
+                    $('#browse-path').val($(this).data('path'));
+                    if ($(this).hasClass('folder')) {
+                        browsePath($(this).data('key'), null, filter_ext)
+                    }
+                });
+            }
+        });
+    }
+
 $(document).ready(function() {
 
     // Javascript to enable link to tab
@@ -2077,6 +2158,7 @@ $(document).ready(function() {
     initConfigCheckbox('#https_create_cert');
     initConfigCheckbox('#check_github');
     initConfigCheckbox('#monitor_pms_updates');
+    initConfigCheckbox('#monitor_remote_access');
     initConfigCheckbox('#newsletter_self_hosted');
 
     $('#menu_link_shutdown').click(function() {
@@ -2472,9 +2554,7 @@ $(document).ready(function() {
         $("#token_verify").html('<i class="fa fa-refresh fa-spin"></i>').fadeIn('fast');
     }
     function OAuthSuccessCallback(authToken) {
-        var x_plex_headers = getPlexHeaders();
         $("#pms_token").val(authToken);
-        $("#pms_uuid").val(x_plex_headers['X-Plex-Client-Identifier']);
         $("#token_verify").html('<i class="fa fa-check"></i>').fadeIn('fast');
         getServerOptions(authToken);
     }
@@ -2929,54 +3009,46 @@ $(document).ready(function() {
         openPlexXML('/api/resources', true, {includeHttps: 1});
     });
 
-    if ("${kwargs.get('install_geoip')}" === 'true') {
-        gotoSetting('3rd_party_apis', 'geoip_db')
-    }
-
-    if ("${config['geoip_db_installed']}" > "0") {
-        $("#geoip_db_updated").text(moment("${config['geoip_db_installed']}", "X").fromNow());
-    }
-
-    $("#install_geoip_db").click(function () {
-        var maxmind_license_key = $("#maxmind_license_key");
-        maxmind_license_key.val($.trim(maxmind_license_key.val()));
-        if (maxmind_license_key.val() === "") {
-            maxmind_license_key.focus();
-            showMsg('<i class="fa fa-exclamation-circle"></i> Maxmind License Key is required.', false, true, 5000, true);
-            return false;
-        } else if (!(saveSettings())) {
-            return false;
-        }
-        var msg = 'Are you sure you want to install the GeoLite2 database?<br /><br />' +
-            'The database is used to lookup IP address geolocation info.<br />' +
-            'The database will be downloaded from <a href="${anon_url("https://dev.maxmind.com/geoip/geoip2/geolite2/")}" target="_blank">MaxMind</a>, <br />' +
-            'and requires <strong>100MB</strong> of free space to install.<br />';
-        var url = 'install_geoip_db';
-        if ($(this).text() === 'Update') {
-            url += '?update=true';
-        }
-        confirmAjaxCall(url, msg, null, 'Installing GeoLite2 database.', function (result) {
-            if (result.result === "success") {
-                $('#install_geoip_db').text('Update');
-                $('#uninstall_geoip_db').prop('disabled', false);
-                $('#geoip_db_updated').text(moment(result.updated, "X").fromNow());
+    var tautulli_news = $('#tautulli-news')
+    $.ajax({
+        url: 'https://tautulli.com/news/tautulli-news.json',
+        type: 'GET',
+        dataType: 'json',
+        cache: false,
+        async: true,
+        success: function (data) {
+            if (data) {
+                var now = moment().endOf('day');
+                var news = $('<ul/>').addClass('accordion list-unstyled')
+                $.each(data, function (index, news_item) {
+                    var date = moment(news_item.date, "YYYY-MM-DD");
+                    if (index >= 5) { return false; }
+                    var header = $('<div/>').addClass('link').html(
+                        '<span class="toggle-left"><i class="fa fa-newspaper fa-fw"></i></span>' +
+                        '<span class="news-title">' + news_item.title + '</span>' +
+                        '<span class="toggle-right"><i class="fa fa-chevron-down fa-fw"></i></span>' +
+                        '<span class="news-date toggle-right">' + date.format($('#date_format').val()) + '</span>');
+                    var subtitle = $('<span/>').addClass('news-subtitle').html(news_item.subtitle);
+                    var body = $('<span/>').addClass('news-body').html(news_item.body);
+                    var content = $('<div/>').addClass('submenu');
+                    if (news_item.subtitle) { content.append(subtitle); }
+                    content.append(body);
+                    var li = $('<li/>').append(header).append(content)
+                    if (index === 0 && Math.abs(now.diff(date, 'days')) < 7) {
+                        li.addClass('open');
+                        content.css('display', 'block');
+                    }
+                    news.append(li)
+                });
+                tautulli_news.html(news);
+                var accordion_news = new Accordion(news, false);
+            } else {
+                tautulli_news.html('<p class="help-block"><i class="fa fa-check"></i>&nbsp; No news available.</p>')
             }
-            getSchedulerTable();
-        });
-    });
-
-    $("#uninstall_geoip_db").click(function () {
-        var msg = 'Are you sure you want to uninstall the GeoLite2 database?<br /><br />' +
-            'You will not be able to lookup IP address geolocation info.';
-        var url = 'uninstall_geoip_db';
-        confirmAjaxCall(url, msg, null, 'Uninstalling GeoLite2 database.', function (result) {
-            if (result.result === "success") {
-                $('#install_geoip_db').text('Install');
-                $('#uninstall_geoip_db').prop('disabled', true);
-                $('#geoip_db_updated').text('never');
-            }
-            getSchedulerTable();
-        });
+        },
+        error: function () {
+            tautulli_news.html('<p class="help-block"><i class="fa fa-exclamation-triangle"></i>&nbsp; Failed to retrieve news.</p>')
+        }
     });
 });
 </script>

data/interfaces/default/user.html

@@ -582,9 +582,9 @@ DOCUMENTATION :: END
                         success: function (data) {
                             var msg = "History deleted";
                             showMsg(msg, false, true, 2000);
+                            history_table.draw();
                         }
                     });
-                    history_table.draw();
                 });
             }
 

data/interfaces/default/user_player_stats.html

@@ -27,7 +27,7 @@ DOCUMENTATION :: END
             <div id="user-player-image-${a['result_id']}">
                 <div class="user-player-instance-box svg-icon platform-${a['platform_name']}"></div>
             </div>
-            <div class="user-player-instance-name">
+            <div class="user-player-instance-name" title="${a['player_name']}">
                 ${a['player_name']}
             </div>
             <div class="user-player-instance-playcount">

data/interfaces/default/welcome.html

@@ -203,7 +203,7 @@
                     <h3>Database Import</h3>
                     <div class="wizard-input-section">
                         <p class="help-block">
-                            If you have an existing PlexWatch/Plexivity database, you can import the data into Tautulli.
+                            If you have an existing Tautulli, PlexWatch, or Plexivity database, you can import the data into Tautulli.
                         </p>
                         <p class="help-block">
                             To import a database, navigate to the <strong>Settings</strong> page
@@ -216,7 +216,8 @@
                         <input type="checkbox" name="first_run" id="first_run" value="1" checked>
                         <input type="checkbox" name="group_history_tables" id="group_history_tables" value="1" checked>
                         <input type="checkbox" name="history_table_activity" id="history_table_activity" value="1" checked>
-                        <input type="checkbox" name="win_sys_tray" id="win_sys_tray" value="1" checked>
+                        <input type="checkbox" name="sys_tray_icon" id="sys_tray_icon" value="1" checked>
+                        <input type="checkbox" name="launch_startup" id="launch_startup" value="1" checked>
                         <input type="checkbox" name="launch_browser" id="launch_browser" value="1" checked>
                         <input type="checkbox" name="api_enabled" id="api_enabled" value="1" checked>
                         <input type="checkbox" name="refresh_users_on_startup" id="refresh_users_on_startup" value="1" checked>
@@ -494,7 +495,7 @@ $(document).ready(function() {
             var pms_ssl = $("#pms_ssl").val();
             var pms_is_remote = $("#pms_is_remote").val();
             if ((pms_ip !== '') || (pms_port !== '')) {
-                $("#pms-verify-status").html('<i class="fa fa-refresh fa-spin"></i>&nbsp; Validating server...');
+                $("#pms-verify-status").html('<i class="fa fa-refresh fa-spin"></i>&nbsp; Verifying server...');
                 $('#pms-verify-status').fadeIn('fast');
                 $.ajax({
                     url: 'get_server_id',
@@ -509,7 +510,7 @@ $(document).ready(function() {
                     async: true,
                     timeout: 5000,
                     error: function (jqXHR, textStatus, errorThrown) {
-                        $("#pms-verify-status").html('<i class="fa fa-exclamation-circle"></i>&nbsp; This is not a Plex Server!');
+                        $("#pms-verify-status").html('<i class="fa fa-exclamation-circle"></i>&nbsp; Error verifying server: ' + textStatus);
                         $('#pms-verify-status').fadeIn('fast');
                     },
                     success: function(xhr, status) {

init-scripts/init-alt.freebsd

@@ -1,88 +0,0 @@
-#!/bin/sh
-#
-# PROVIDE: tautulli
-# REQUIRE: tautulli
-# KEYWORD: shutdown
-#
-# Add the following lines to /etc/rc.conf.local or /etc/rc.conf
-# to enable this service:
-#
-# tautulli_enable (bool):     Set to NO by default.
-#                       Set it to YES to enable it.
-# tautulli_user:  The user account Tautulli daemon runs as what
-#                       you want it to be. It uses 'tautulli' user by
-#                       default. Do not sets it as empty or it will run
-#                       as root.
-# tautulli_dir:       Directory where Tautulli lives.
-#                       Default: /usr/local/share/Tautulli
-# tautulli_chdir:  Change to this directory before running Tautulli.
-#     Default is same as tautulli_dir.
-# tautulli_pid:  The name of the pidfile to create.
-#     Default is tautulli.pid in tautulli_dir.
-
-. /etc/rc.subr
-
-name="tautulli"
-rcvar=${name}_enable
-
-load_rc_config ${name}
-
-: ${tautulli_enable:="NO"}
-: ${tautulli_user:="tautulli"}
-: ${tautulli_dir:="/usr/local/share/Tautulli"}
-: ${tautulli_chdir:="${tautulli_dir}"}
-: ${tautulli_pid:="${tautulli_dir}/tautulli.pid"}
-: ${tautulli_conf:="${tautulli_dir}/config.ini"}
-
-WGET="/usr/local/bin/wget" # You need wget for this script to safely shutdown Tautulli.
-if [ -e "${tautulli_conf}" ]; then
-        HOST=`grep -A64 "\[General\]" "${tautulli_conf}"|egrep "^http_host"|perl -wple 's/^http_host = (.*)$/$1/'`
-        PORT=`grep -A64 "\[General\]" "${tautulli_conf}"|egrep "^http_port"|perl -wple 's/^http_port = (.*)$/$1/'`
-fi
-
-status_cmd="${name}_status"
-stop_cmd="${name}_stop"
-
-command="${tautulli_dir}/Tautulli.py"
-command_args="--daemon --quiet --nolaunch --port ${PORT} --pidfile ${tautulli_pid} --config ${tautulli_conf}"
-
-# Check for wget and refuse to start without it.
-if [ ! -x "${WGET}" ]; then
-  warn "Tautulli not started: You need wget to safely shut down Tautulli."
-  exit 1
-fi
-
-# Ensure user is root when running this script.
-if [ `id -u` != "0" ]; then
-  echo "Oops, you should be root before running this!"
-  exit 1
-fi
-
-verify_tautulli_pid() {
-    # Make sure the pid corresponds to the Tautulli process.
-    pid=`cat ${tautulli_pid} 2>/dev/null`
-    ps -p ${pid} | grep -q "python ${tautulli_dir}/Tautulli.py"
-    return $?
-}
-
-# Try to stop Tautulli cleanly by calling shutdown over http.
-tautulli_stop() {
-    if [ ! -e "${tautulli_conf}" ]; then
-        echo "Tautulli' settings file does not exist. Try starting Tautulli, as this should create the file."
-        exit 1
-    fi
-    echo "Stopping $name"
-    verify_tautulli_pid
-    ${WGET} -O - -q --user=${SBUSR} --password=${SBPWD} "http://${HOST}:${PORT}/shutdown/" >/dev/null
-    
-    if [ -n "${pid}" ]; then
-      wait_for_pids ${pid}
-      echo "Stopped $name"
-    fi
-}
-
-tautulli_status() {
-    verify_tautulli_pid && echo "$name is running as ${pid}" || echo "$name is not running"
-}
-
-run_rc_command "$1"

init-scripts/init.fedora.centos.service

@@ -1,76 +0,0 @@
-#!/bin/sh
-#
-### BEGIN INIT INFO
-# Provides:          Tautulli
-# Required-Start:    $all
-# Required-Stop:     $all
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: starts Tautulli
-# Description:       starts Tautulli
-### END INIT INFO
-
-# Source function library.
-. /etc/init.d/functions
-
-## Variables
-prog=tautulli
-lockfile=/var/lock/subsys/$prog
-homedir=/opt/Tautulli
-datadir=/opt/Tautulli
-configfile=/opt/Tautulli/config.ini
-pidfile=/var/run/tautulli.pid
-nice=
-# The following line must point to your Python 2.7 install
-python27=/usr/src/Python-2.7.11/python
-##
-
-options=" --daemon --config $configfile --pidfile $pidfile --datadir $datadir --nolaunch --quiet"
-
-start() {
-        # Start daemon.
-        echo -n $"Starting $prog: "
-        daemon --pidfile=$pidfile $nice $python27 $homedir/Tautulli.py $options
-        RETVAL=$?
-        echo
-        [ $RETVAL -eq 0 ] && touch $lockfile
-        return $RETVAL
-}
-
-stop() {
-        echo -n $"Shutting down $prog: "
-        killproc -p $pidfile $python27
-        RETVAL=$?
-        echo
-        [ $RETVAL -eq 0 ] && rm -f $lockfile
-        return $RETVAL
-}
-
-# See how we were called.
-case "$1" in
-  start)
-        start
-        ;;
-  stop)
-        stop
-        ;;
-  status)
-        status $prog
-        ;;
-  restart|force-reload)
-        stop
-        start
-        ;;
-  try-restart|condrestart)
-        if status $prog > /dev/null; then
-            stop
-            start
-        fi
-        ;;
-  reload)
-        exit 3
-        ;;
-  *)
-        echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}"
-        exit 2
-esac

init-scripts/init.freebsd

@@ -51,7 +51,7 @@ verify_tautulli_pid() {
     # Make sure the pid corresponds to the Tautulli process.
     if [ -f ${tautulli_pid} ]; then
       pid=`cat ${tautulli_pid} 2>/dev/null`
-      ps -p ${pid} | grep -q "python2 ${tautulli_dir}/Tautulli.py"
+      ps -p ${pid} | grep -q "python ${tautulli_dir}/Tautulli.py"
       return $?
     else
       return 0
@@ -60,7 +60,7 @@ verify_tautulli_pid() {
 
 # Try to stop Tautulli cleanly by sending SIGTERM
 tautulli_stop() {
-    echo "Stopping $name"
+    echo "Stopping $name."
     verify_tautulli_pid
     if [ -n "${pid}" ]; then
       kill ${pid}

init-scripts/init.freenas

@@ -1,81 +0,0 @@
-#!/bin/sh
-#
-# PROVIDE: tautulli
-# REQUIRE: DAEMON tautulli
-# KEYWORD: shutdown
-#
-# Add the following lines to /etc/rc.conf.local or /etc/rc.conf
-# to enable this service:
-#
-# tautulli_enable (bool): Set to NO by default.
-#           Set it to YES to enable it.
-# tautulli_user:  The user account Tautulli daemon runs as what
-#           you want it to be. It uses 'tautulli' user by
-#           default. Do not sets it as empty or it will run
-#           as root.
-# tautulli_dir:   Directory where Tautulli lives.
-#           Default: /usr/local/share/Tautulli
-# tautulli_chdir:  Change to this directory before running Tautulli.
-#     Default is same as tautulli_dir.
-# tautulli_pid:  The name of the pidfile to create.
-#     Default is tautulli.pid in tautulli_dir.
-PATH="/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
-
-. /etc/rc.subr
-
-name="tautulli"
-rcvar=${name}_enable
-
-load_rc_config ${name}
-
-: ${tautulli_enable:="NO"}
-: ${tautulli_user:="tautulli"}
-: ${tautulli_dir:="/usr/local/share/Tautulli"}
-: ${tautulli_chdir:="${tautulli_dir}"}
-: ${tautulli_pid:="${tautulli_dir}/tautulli.pid"}
-: ${tautulli_flags:=""}
-
-status_cmd="${name}_status"
-stop_cmd="${name}_stop"
-
-command="${tautulli_dir}/Tautulli.py"
-command_args="--daemon --pidfile ${tautulli_pid} --quiet --nolaunch ${tautulli_flags}"
-
-# Ensure user is root when running this script.
-if [ `id -u` != "0" ]; then
-  echo "Oops, you should be root before running this!"
-  exit 1
-fi
-
-verify_tautulli_pid() {
-    # Make sure the pid corresponds to the Tautulli process.
-    if [ -f ${tautulli_pid} ]; then
-      pid=`cat ${tautulli_pid} 2>/dev/null`
-      ps -p ${pid} | grep -q "python2 ${tautulli_dir}/Tautulli.py"
-      return $?
-    else
-      return 0
-    fi
-}
-
-# Try to stop Tautulli cleanly by sending SIGTERM
-tautulli_stop() {
-    echo "Stopping $name."
-    verify_tautulli_pid
-    if [ -n "${pid}" ]; then
-      kill ${pid}
-      wait_for_pids ${pid}
-      echo "Stopped."
-    fi
-}
-
-tautulli_status() {
-    verify_tautulli_pid
-    if [ -n "${pid}" ]; then
-      echo "$name is running as ${pid}."
-    else
-      echo "$name is not running."
-    fi
-}
-
-run_rc_command "$1"

init-scripts/init.freenas

@@ -0,0 +1 @@
+init.freebsd

init-scripts/init.solaris11

@@ -1,47 +0,0 @@
-<?xml version="1.0"?>
-<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
-<!--
-        Created by Manifold
---><service_bundle type="manifest" name="tautulli">
-
-    <service name="application/tautulli" type="service" version="1">
-
-        <create_default_instance enabled="true"/>
-
-        <single_instance/>
-
-        <dependency name="network" grouping="require_all" restart_on="error" type="service">
-            <service_fmri value="svc:/milestone/network:default"/>
-        </dependency>
-
-        <dependency name="filesystem" grouping="require_all" restart_on="error" type="service">
-            <service_fmri value="svc:/system/filesystem/local"/>
-        </dependency>
-
-        <method_context>
-            <method_credential user="tautulli" group="nogroup"/>
-        </method_context>
-
-        <exec_method type="method" name="start" exec="python /opt/Tautulli/Tautulli.py --daemon --quiet --nolaunch" timeout_seconds="60"/>
-
-        <exec_method type="method" name="stop" exec=":kill" timeout_seconds="60"/>
-
-        <property_group name="startd" type="framework">
-            <propval name="duration" type="astring" value="contract"/>
-            <propval name="ignore_error" type="astring" value="core,signal"/>
-        </property_group>
-
-
-        <stability value="Evolving"/>
-
-        <template>
-            <common_name>
-                <loctext xml:lang="C">
-                    Tautulli
-                </loctext>
-            </common_name>
-        </template>
-
-    </service>
-
-</service_bundle>

init-scripts/init.ubuntu

@@ -1,209 +0,0 @@
-#!/bin/sh
-#
-## Don't edit this file
-## Edit user configuation in /etc/default/tautulli to change
-##
-## Make sure init script is executable
-## sudo chmod +x /path/to/init.ubuntu
-##
-## Install the init script
-## sudo ln -s /path/to/init.ubuntu /etc/init.d/tautulli
-##
-## Create the tautulli daemon user:
-## sudo adduser --system --no-create-home tautulli
-##
-## Make sure /opt/Tautulli is owned by the tautulli user
-## sudo chown tautulli:nogroup -R /opt/Tautulli
-##
-## Touch the default file to stop the warning message when starting
-## sudo touch /etc/default/tautulli
-##
-## To start Tautulli automatically
-## sudo  update-rc.d tautulli defaults
-##
-## To start/stop/restart Tautulli
-## sudo service tautulli start
-## sudo service tautulli stop
-## sudo service tautulli restart
-##
-## TAUTULLI_USER=         #$RUN_AS, username to run Tautulli under, the default is tautulli
-## TAUTULLI_HOME=         #$APP_PATH, the location of Tautulli.py, the default is /opt/Tautulli
-## TAUTULLI_DATA=         #$DATA_DIR, the location of plexpy.db, cache, logs, the default is /opt/Tautulli
-## TAUTULLI_PIDFILE=      #$PID_FILE, the location of tautulli.pid, the default is /var/run/tautulli/tautulli.pid
-## PYTHON_BIN=      #$DAEMON, the location of the python binary, the default is /usr/bin/python
-## TAUTULLI_OPTS=         #$EXTRA_DAEMON_OPTS, extra cli option for Tautulli, i.e. " --config=/home/Tautulli/config.ini"
-## SSD_OPTS=        #$EXTRA_SSD_OPTS, extra start-stop-daemon option like " --group=users"
-## TAUTULLI_PORT=         #$PORT_OPTS, hardcoded port for the webserver, overrides value in config.ini
-##
-## EXAMPLE if want to run as different user
-## add TAUTULLI_USER=username to /etc/default/tautulli
-## otherwise default tautulli is used
-#
-### BEGIN INIT INFO
-# Provides:          tautulli
-# Required-Start:    $local_fs $network $remote_fs
-# Required-Stop:     $local_fs $network $remote_fs
-# Should-Start:      $NetworkManager
-# Should-Stop:       $NetworkManager
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: starts instance of Tautulli
-# Description:       starts instance of Tautulli using start-stop-daemon
-### END INIT INFO
-
-# Script name
-NAME=tautulli
-
-# App name
-DESC=Tautulli
-
-SETTINGS_LOADED=FALSE
-
-. /lib/lsb/init-functions
-
-# Source Tautulli configuration
-if [ -f /etc/default/tautulli ]; then
-    SETTINGS=/etc/default/tautulli
-else
-    log_warning_msg "/etc/default/tautulli not found using default settings.";
-fi
-
-check_retval() {
-    if [ $? -eq 0 ]; then
-        log_end_msg 0
-        return 0
-    else
-        log_end_msg 1
-        exit 1
-    fi
-}
-
-load_settings() {
-    if [ $SETTINGS_LOADED != "TRUE" ]; then
-        . $SETTINGS
-
-        ## The defaults
-        # Run as username
-        RUN_AS=${TAUTULLI_USER-tautulli}
-
-        # Path to app TAUTULLI_HOME=path_to_app_Tautulli.py
-        APP_PATH=${TAUTULLI_HOME-/opt/Tautulli}
-
-        # Data directory where plexpy.db, cache and logs are stored
-        DATA_DIR=${TAUTULLI_DATA-/opt/Tautulli}
-
-        # Path to store PID file
-        PID_FILE=${TAUTULLI_PIDFILE-/var/run/tautulli/tautulli.pid}
-
-        # Path to python bin
-        DAEMON=${PYTHON_BIN-/usr/bin/python}
-
-        # Extra daemon option like: TAUTULLI_OPTS=" --config=/home/Tautulli/config.ini"
-        EXTRA_DAEMON_OPTS=${TAUTULLI_OPTS-}
-
-        # Extra start-stop-daemon option like START_OPTS=" --group=users"
-        EXTRA_SSD_OPTS=${SSD_OPTS-}
-
-        # Hardcoded port to run on, overrides config.ini settings
-        [ -n "$TAUTULLI_PORT" ] && {
-            PORT_OPTS=" --port=${TAUTULLI_PORT} "
-        }
-
-        DAEMON_OPTS=" Tautulli.py --quiet --daemon --nolaunch --pidfile=${PID_FILE} --datadir=${DATA_DIR} ${PORT_OPTS}${EXTRA_DAEMON_OPTS}"
-
-        SETTINGS_LOADED=TRUE
-    fi
-
-    [ -x $DAEMON ] || {
-        log_warning_msg "$DESC: Can't execute daemon, aborting. See $DAEMON";
-        return 1;}
-
-    return 0
-}
-
-load_settings || exit 0
-
-is_running () {
-    # returns 1 when running, else 0.
-    if [ -e $PID_FILE ]; then
-      PID=`cat $PID_FILE`
-
-      RET=$?
-      [ $RET -gt 1 ] && exit 1 || return $RET
-    else
-      return 1
-    fi
-}
-
-handle_pid () {
-    PID_PATH=`dirname $PID_FILE`
-    [ -d $PID_PATH ] || mkdir -p $PID_PATH && chown -R $RUN_AS $PID_PATH > /dev/null || {
-        log_warning_msg "$DESC: Could not create $PID_FILE, See $SETTINGS, aborting.";
-        return 1;}
-
-    if [ -e $PID_FILE ]; then
-        PID=`cat $PID_FILE`
-        if ! kill -0 $PID > /dev/null 2>&1; then
-            log_warning_msg "Removing stale $PID_FILE"
-            rm $PID_FILE
-        fi
-    fi
-}
-
-handle_datadir () {
-    [ -d $DATA_DIR ] || mkdir -p $DATA_DIR && chown -R $RUN_AS $DATA_DIR > /dev/null || {
-        log_warning_msg "$DESC: Could not create $DATA_DIR, See $SETTINGS, aborting.";
-        return 1;}
-}
-
-handle_updates () {
-    chown -R $RUN_AS $APP_PATH > /dev/null || {
-        log_warning_msg "$DESC: $APP_PATH not writable by $RUN_AS for web-updates";
-        return 0; }
-}
-
-start_tautulli () {
-    handle_pid
-    handle_datadir
-    handle_updates
-    if ! is_running; then
-        log_daemon_msg "Starting $DESC"
-        start-stop-daemon -o -d $APP_PATH -c $RUN_AS --start $EXTRA_SSD_OPTS --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS
-        check_retval
-    else
-        log_success_msg "$DESC: already running (pid $PID)"
-    fi
-}
-
-stop_tautulli () {
-    if is_running; then
-        log_daemon_msg "Stopping $DESC"
-        start-stop-daemon -o --stop --pidfile $PID_FILE --retry 15
-        check_retval
-    else
-        log_success_msg "$DESC: not running"
-    fi
-}
-
-case "$1" in
-    start)
-        start_tautulli
-        ;;
-    stop)
-        stop_tautulli
-        ;;
-    restart|force-reload)
-        stop_tautulli
-        start_tautulli
-        ;;
-    status)
-        status_of_proc -p "$PID_FILE" "$DAEMON" "$DESC"
-        ;;
-    *)
-        N=/etc/init.d/$NAME
-        echo "Usage: $N {start|stop|restart|force-reload|status}" >&2
-        exit 1
-        ;;
-esac
-
-exit 0

init-scripts/init.upstart

@@ -1,18 +0,0 @@
-# tautulli
-#
-# This is a session/user job. Install this file into /usr/share/upstart/sessions
-# if Tautulli is installed system wide, and into $XDG_CONFIG_HOME/upstart if
-# Tautulli is installed per user. Change the executable path appropiately.
-
-start on desktop-start
-stop on desktop-end
-
-env CONFIG=""$XDG_CONFIG_HOME"/Tautulli"
-env DATA=""$XDG_DATA_HOME"/Tautulli"
-
-pre-start script
-    [ -d "$CONFIG" ] || mkdir -p "$CONFIG"
-    [ -d "$DATA" ] || mkdir -p "$DATA"
-end script
-
-exec Tautulli.py --nolaunch --config "$CONFIG"/config.ini --datadir "$DATA"

lib/UniversalAnalytics/HTTPLog.py

@@ -10,7 +10,7 @@
 
 
 import sys, re, os
-from cStringIO import StringIO
+from io import StringIO
 
 
 
@@ -116,6 +116,6 @@ def consume(outbuffer = None): # Capture standard output
 
 if __name__ == '__main__':
     consume(sys.stdout).write(sys.stdin.read())
-    print '\n'
+    print('\n')
 
 # vim: set nowrap tabstop=4 shiftwidth=4 softtabstop=0 expandtab textwidth=0 filetype=python foldmethod=indent foldcolumn=4

lib/UniversalAnalytics/Tracker.py

@@ -1,16 +1,7 @@
-###############################################################################
-# Universal Analytics for Python
-# Copyright (c) 2013, Analytics Pros
-# 
-# This project is free software, distributed under the BSD license. 
-# Analytics Pros offers consulting and integration services if your firm needs 
-# assistance in strategy, implementation, or auditing existing work.
-###############################################################################
-
-from urllib2 import urlopen, build_opener, install_opener
-from urllib2 import Request, HTTPSHandler
-from urllib2 import URLError, HTTPError
-from urllib import urlencode
+from future.moves.urllib.request import urlopen, build_opener, install_opener
+from future.moves.urllib.request import Request, HTTPSHandler
+from future.moves.urllib.error import URLError, HTTPError
+from future.moves.urllib.parse import urlencode
 
 import random
 import datetime
@@ -24,8 +15,8 @@ def generate_uuid(basedata=None):
     """ Provides a _random_ UUID with no input, or a UUID4-format MD5 checksum of any input data provided """
     if basedata is None:
         return str(uuid.uuid4())
-    elif isinstance(basedata, basestring):
-        checksum = hashlib.md5(basedata).hexdigest()
+    elif isinstance(basedata, str):
+        checksum = hashlib.md5(str(basedata).encode('utf-8')).hexdigest()
         return '%8s-%4s-%4s-%4s-%12s' % (
         checksum[0:8], checksum[8:12], checksum[12:16], checksum[16:20], checksum[20:32])
 
@@ -44,7 +35,7 @@ class Time(datetime.datetime):
     def to_unix(cls, timestamp):
         """ Wrapper over time module to produce Unix epoch time as a float """
         if not isinstance(timestamp, datetime.datetime):
-            raise TypeError, 'Time.milliseconds expects a datetime object'
+            raise TypeError('Time.milliseconds expects a datetime object')
         base = time.mktime(timestamp.timetuple())
         return base
 
@@ -86,14 +77,14 @@ class HTTPRequest(object):
     def fixUTF8(cls, data):  # Ensure proper encoding for UA's servers...
         """ Convert all strings to UTF-8 """
         for key in data:
-            if isinstance(data[key], basestring):
+            if isinstance(data[key], str):
                 data[key] = data[key].encode('utf-8')
         return data
 
     # Apply stored properties to the given dataset & POST to the configured endpoint
     def send(self, data):
         request = Request(
-            self.endpoint + '?' + urlencode(self.fixUTF8(data)),
+            self.endpoint + '?' + urlencode(self.fixUTF8(data)).encode('utf-8'),
             headers={
                 'User-Agent': self.user_agent
             }
@@ -121,7 +112,7 @@ class HTTPPost(HTTPRequest):
     def send(self, data):
         request = Request(
             self.endpoint,
-            data=urlencode(self.fixUTF8(data)),
+            data=urlencode(self.fixUTF8(data)).encode('utf-8'),
             headers={
                 'User-Agent': self.user_agent
             }
@@ -144,26 +135,26 @@ class Tracker(object):
 
     @classmethod
     def coerceParameter(cls, name, value=None):
-        if isinstance(name, basestring) and name[0] == '&':
+        if isinstance(name, str) and name[0] == '&':
             return name[1:], str(value)
         elif name in cls.parameter_alias:
             typecast, param_name = cls.parameter_alias.get(name)
             return param_name, typecast(value)
         else:
-            raise KeyError, 'Parameter "{0}" is not recognized'.format(name)
+            raise KeyError('Parameter "{0}" is not recognized'.format(name))
 
     def payload(self, data):
-        for key, value in data.iteritems():
+        for key, value in data.items():
             try:
                 yield self.coerceParameter(key, value)
             except KeyError:
                 continue
 
     option_sequence = {
-        'pageview': [(basestring, 'dp')],
-        'event': [(basestring, 'ec'), (basestring, 'ea'), (basestring, 'el'), (int, 'ev')],
-        'social': [(basestring, 'sn'), (basestring, 'sa'), (basestring, 'st')],
-        'timing': [(basestring, 'utc'), (basestring, 'utv'), (basestring, 'utt'), (basestring, 'utl')]
+        'pageview': [(str, 'dp')],
+        'event': [(str, 'ec'), (str, 'ea'), (str, 'el'), (int, 'ev')],
+        'social': [(str, 'sn'), (str, 'sa'), (str, 'st')],
+        'timing': [(str, 'utc'), (str, 'utv'), (str, 'utt'), (str, 'utl')]
     }
 
     @classmethod
@@ -232,7 +223,7 @@ class Tracker(object):
                 for key, val in self.payload(item):
                     data[key] = val
 
-        for k, v in self.params.iteritems():  # update only absent parameters
+        for k, v in self.params.items():  # update only absent parameters
             if k not in data:
                 data[k] = v
 
@@ -247,13 +238,13 @@ class Tracker(object):
     # Setting persistent attibutes of the session/hit/etc (inc. custom dimensions/metrics)
     def set(self, name, value=None):
         if isinstance(name, dict):
-            for key, value in name.iteritems():
+            for key, value in name.items():
                 try:
                     param, value = self.coerceParameter(key, value)
                     self.params[param] = value
                 except KeyError:
                     pass
-        elif isinstance(name, basestring):
+        elif isinstance(name, str):
             try:
                 param, value = self.coerceParameter(name, value)
                 self.params[param] = value
@@ -277,7 +268,7 @@ class Tracker(object):
 def safe_unicode(obj):
     """ Safe convertion to the Unicode string version of the object """
     try:
-        return unicode(obj)
+        return str(obj)
     except UnicodeDecodeError:
         return obj.decode('utf-8')
 
@@ -380,7 +371,7 @@ for i in range(0, 5):
 # Enhanced Ecommerce
 Tracker.alias(str, 'pa')  # Product action
 Tracker.alias(str, 'tcc')  # Coupon code
-Tracker.alias(unicode, 'pal')  # Product action list
+Tracker.alias(str, 'pal')  # Product action list
 Tracker.alias(int, 'cos')  # Checkout step
 Tracker.alias(str, 'col')  # Checkout step option
 
@@ -388,10 +379,10 @@ Tracker.alias(str, 'promoa')  # Promotion action
 
 for product_index in range(1, MAX_EC_PRODUCTS):
     Tracker.alias(str, 'pr{0}id'.format(product_index))  # Product SKU
-    Tracker.alias(unicode, 'pr{0}nm'.format(product_index))  # Product name
-    Tracker.alias(unicode, 'pr{0}br'.format(product_index))  # Product brand
-    Tracker.alias(unicode, 'pr{0}ca'.format(product_index))  # Product category
-    Tracker.alias(unicode, 'pr{0}va'.format(product_index))  # Product variant
+    Tracker.alias(str, 'pr{0}nm'.format(product_index))  # Product name
+    Tracker.alias(str, 'pr{0}br'.format(product_index))  # Product brand
+    Tracker.alias(str, 'pr{0}ca'.format(product_index))  # Product category
+    Tracker.alias(str, 'pr{0}va'.format(product_index))  # Product variant
     Tracker.alias(str, 'pr{0}pr'.format(product_index))  # Product price
     Tracker.alias(int, 'pr{0}qt'.format(product_index))  # Product quantity
     Tracker.alias(str, 'pr{0}cc'.format(product_index))  # Product coupon code
@@ -403,10 +394,10 @@ for product_index in range(1, MAX_EC_PRODUCTS):
 
     for list_index in range(1, MAX_EC_LISTS):
         Tracker.alias(str, 'il{0}pi{1}id'.format(list_index, product_index))  # Product impression SKU
-        Tracker.alias(unicode, 'il{0}pi{1}nm'.format(list_index, product_index))  # Product impression name
-        Tracker.alias(unicode, 'il{0}pi{1}br'.format(list_index, product_index))  # Product impression brand
-        Tracker.alias(unicode, 'il{0}pi{1}ca'.format(list_index, product_index))  # Product impression category
-        Tracker.alias(unicode, 'il{0}pi{1}va'.format(list_index, product_index))  # Product impression variant
+        Tracker.alias(str, 'il{0}pi{1}nm'.format(list_index, product_index))  # Product impression name
+        Tracker.alias(str, 'il{0}pi{1}br'.format(list_index, product_index))  # Product impression brand
+        Tracker.alias(str, 'il{0}pi{1}ca'.format(list_index, product_index))  # Product impression category
+        Tracker.alias(str, 'il{0}pi{1}va'.format(list_index, product_index))  # Product impression variant
         Tracker.alias(int, 'il{0}pi{1}ps'.format(list_index, product_index))  # Product impression position
         Tracker.alias(int, 'il{0}pi{1}pr'.format(list_index, product_index))  # Product impression price
 
@@ -417,11 +408,11 @@ for product_index in range(1, MAX_EC_PRODUCTS):
                                                         custom_index))  # Product impression custom metric
 
 for list_index in range(1, MAX_EC_LISTS):
-    Tracker.alias(unicode, 'il{0}nm'.format(list_index))  # Product impression list name
+    Tracker.alias(str, 'il{0}nm'.format(list_index))  # Product impression list name
 
 for promotion_index in range(1, MAX_EC_PROMOTIONS):
     Tracker.alias(str, 'promo{0}id'.format(promotion_index))  # Promotion ID
-    Tracker.alias(unicode, 'promo{0}nm'.format(promotion_index))  # Promotion name
+    Tracker.alias(str, 'promo{0}nm'.format(promotion_index))  # Promotion name
     Tracker.alias(str, 'promo{0}cr'.format(promotion_index))  # Promotion creative
     Tracker.alias(str, 'promo{0}ps'.format(promotion_index))  # Promotion position
 

lib/UniversalAnalytics/__init__.py

@@ -1 +1 @@
-import Tracker
+from . import Tracker

lib/appdirs.py

@@ -0,0 +1,608 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2005-2010 ActiveState Software Inc.
+# Copyright (c) 2013 Eddy Petrișor
+
+"""Utilities for determining application-specific dirs.
+
+See <http://github.com/ActiveState/appdirs> for details and usage.
+"""
+# Dev Notes:
+# - MSDN on where to store app data files:
+#   http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120
+# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html
+# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+
+__version_info__ = (1, 4, 3)
+__version__ = '.'.join(map(str, __version_info__))
+
+
+import sys
+import os
+
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    unicode = str
+
+if sys.platform.startswith('java'):
+    import platform
+    os_name = platform.java_ver()[3][0]
+    if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc.
+        system = 'win32'
+    elif os_name.startswith('Mac'): # "Mac OS X", etc.
+        system = 'darwin'
+    else: # "Linux", "SunOS", "FreeBSD", etc.
+        # Setting this to "linux2" is not ideal, but only Windows or Mac
+        # are actually checked for and the rest of the module expects
+        # *sys.platform* style strings.
+        system = 'linux2'
+else:
+    system = sys.platform
+
+
+
+def user_data_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user data directories are:
+        Mac OS X:               ~/Library/Application Support/<AppName>
+        Unix:                   ~/.local/share/<AppName>    # or in $XDG_DATA_HOME, if defined
+        Win XP (not roaming):   C:\Documents and Settings\<username>\Application Data\<AppAuthor>\<AppName>
+        Win XP (roaming):       C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>
+        Win 7  (not roaming):   C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>
+        Win 7  (roaming):       C:\Users\<username>\AppData\Roaming\<AppAuthor>\<AppName>
+
+    For Unix, we follow the XDG spec and support $XDG_DATA_HOME.
+    That means, by default "~/.local/share/<AppName>".
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA"
+        path = os.path.normpath(_get_win_folder(const))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+    elif system == 'darwin':
+        path = os.path.expanduser('~/Library/Application Support/')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def site_data_dir(appname=None, appauthor=None, version=None, multipath=False):
+    r"""Return full path to the user-shared data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "multipath" is an optional parameter only applicable to *nix
+            which indicates that the entire list of data dirs should be
+            returned. By default, the first item from XDG_DATA_DIRS is
+            returned, or '/usr/local/share/<AppName>',
+            if XDG_DATA_DIRS is not set
+
+    Typical site data directories are:
+        Mac OS X:   /Library/Application Support/<AppName>
+        Unix:       /usr/local/share/<AppName> or /usr/share/<AppName>
+        Win XP:     C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName>
+        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
+        Win 7:      C:\ProgramData\<AppAuthor>\<AppName>   # Hidden, but writeable on Win 7.
+
+    For Unix, this is using the $XDG_DATA_DIRS[0] default.
+
+    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA"))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+    elif system == 'darwin':
+        path = os.path.expanduser('/Library/Application Support')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        # XDG default for $XDG_DATA_DIRS
+        # only first, if multipath is False
+        path = os.getenv('XDG_DATA_DIRS',
+                         os.pathsep.join(['/usr/local/share', '/usr/share']))
+        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)]
+        if appname:
+            if version:
+                appname = os.path.join(appname, version)
+            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+
+        if multipath:
+            path = os.pathsep.join(pathlist)
+        else:
+            path = pathlist[0]
+        return path
+
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_config_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific config dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user config directories are:
+        Mac OS X:               same as user_data_dir
+        Unix:                   ~/.config/<AppName>     # or in $XDG_CONFIG_HOME, if defined
+        Win *:                  same as user_data_dir
+
+    For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME.
+    That means, by default "~/.config/<AppName>".
+    """
+    if system in ["win32", "darwin"]:
+        path = user_data_dir(appname, appauthor, None, roaming)
+    else:
+        path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def site_config_dir(appname=None, appauthor=None, version=None, multipath=False):
+    r"""Return full path to the user-shared data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "multipath" is an optional parameter only applicable to *nix
+            which indicates that the entire list of config dirs should be
+            returned. By default, the first item from XDG_CONFIG_DIRS is
+            returned, or '/etc/xdg/<AppName>', if XDG_CONFIG_DIRS is not set
+
+    Typical site config directories are:
+        Mac OS X:   same as site_data_dir
+        Unix:       /etc/xdg/<AppName> or $XDG_CONFIG_DIRS[i]/<AppName> for each value in
+                    $XDG_CONFIG_DIRS
+        Win *:      same as site_data_dir
+        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
+
+    For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False
+
+    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
+    """
+    if system in ["win32", "darwin"]:
+        path = site_data_dir(appname, appauthor)
+        if appname and version:
+            path = os.path.join(path, version)
+    else:
+        # XDG default for $XDG_CONFIG_DIRS
+        # only first, if multipath is False
+        path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg')
+        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)]
+        if appname:
+            if version:
+                appname = os.path.join(appname, version)
+            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+
+        if multipath:
+            path = os.pathsep.join(pathlist)
+        else:
+            path = pathlist[0]
+    return path
+
+
+def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True):
+    r"""Return full path to the user-specific cache dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "opinion" (boolean) can be False to disable the appending of
+            "Cache" to the base app data dir for Windows. See
+            discussion below.
+
+    Typical user cache directories are:
+        Mac OS X:   ~/Library/Caches/<AppName>
+        Unix:       ~/.cache/<AppName> (XDG default)
+        Win XP:     C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Cache
+        Vista:      C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Cache
+
+    On Windows the only suggestion in the MSDN docs is that local settings go in
+    the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming
+    app data dir (the default returned by `user_data_dir` above). Apps typically
+    put cache data somewhere *under* the given dir here. Some examples:
+        ...\Mozilla\Firefox\Profiles\<ProfileName>\Cache
+        ...\Acme\SuperApp\Cache\1.0
+    OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value.
+    This can be disabled with the `opinion=False` option.
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+            if opinion:
+                path = os.path.join(path, "Cache")
+    elif system == 'darwin':
+        path = os.path.expanduser('~/Library/Caches')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_state_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific state dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user state directories are:
+        Mac OS X:  same as user_data_dir
+        Unix:      ~/.local/state/<AppName>   # or in $XDG_STATE_HOME, if defined
+        Win *:     same as user_data_dir
+
+    For Unix, we follow this Debian proposal <https://wiki.debian.org/XDGBaseDirectorySpecification#state>
+    to extend the XDG spec and support $XDG_STATE_HOME.
+
+    That means, by default "~/.local/state/<AppName>".
+    """
+    if system in ["win32", "darwin"]:
+        path = user_data_dir(appname, appauthor, None, roaming)
+    else:
+        path = os.getenv('XDG_STATE_HOME', os.path.expanduser("~/.local/state"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_log_dir(appname=None, appauthor=None, version=None, opinion=True):
+    r"""Return full path to the user-specific log dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "opinion" (boolean) can be False to disable the appending of
+            "Logs" to the base app data dir for Windows, and "log" to the
+            base cache dir for Unix. See discussion below.
+
+    Typical user log directories are:
+        Mac OS X:   ~/Library/Logs/<AppName>
+        Unix:       ~/.cache/<AppName>/log  # or under $XDG_CACHE_HOME if defined
+        Win XP:     C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Logs
+        Vista:      C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Logs
+
+    On Windows the only suggestion in the MSDN docs is that local settings
+    go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in
+    examples of what some windows apps use for a logs dir.)
+
+    OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA`
+    value for Windows and appends "log" to the user cache dir for Unix.
+    This can be disabled with the `opinion=False` option.
+    """
+    if system == "darwin":
+        path = os.path.join(
+            os.path.expanduser('~/Library/Logs'),
+            appname)
+    elif system == "win32":
+        path = user_data_dir(appname, appauthor, version)
+        version = False
+        if opinion:
+            path = os.path.join(path, "Logs")
+    else:
+        path = user_cache_dir(appname, appauthor, version)
+        version = False
+        if opinion:
+            path = os.path.join(path, "log")
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+class AppDirs(object):
+    """Convenience wrapper for getting application dirs."""
+    def __init__(self, appname=None, appauthor=None, version=None,
+            roaming=False, multipath=False):
+        self.appname = appname
+        self.appauthor = appauthor
+        self.version = version
+        self.roaming = roaming
+        self.multipath = multipath
+
+    @property
+    def user_data_dir(self):
+        return user_data_dir(self.appname, self.appauthor,
+                             version=self.version, roaming=self.roaming)
+
+    @property
+    def site_data_dir(self):
+        return site_data_dir(self.appname, self.appauthor,
+                             version=self.version, multipath=self.multipath)
+
+    @property
+    def user_config_dir(self):
+        return user_config_dir(self.appname, self.appauthor,
+                               version=self.version, roaming=self.roaming)
+
+    @property
+    def site_config_dir(self):
+        return site_config_dir(self.appname, self.appauthor,
+                             version=self.version, multipath=self.multipath)
+
+    @property
+    def user_cache_dir(self):
+        return user_cache_dir(self.appname, self.appauthor,
+                              version=self.version)
+
+    @property
+    def user_state_dir(self):
+        return user_state_dir(self.appname, self.appauthor,
+                              version=self.version)
+
+    @property
+    def user_log_dir(self):
+        return user_log_dir(self.appname, self.appauthor,
+                            version=self.version)
+
+
+#---- internal support stuff
+
+def _get_win_folder_from_registry(csidl_name):
+    """This is a fallback technique at best. I'm not sure if using the
+    registry for this guarantees us the correct answer for all CSIDL_*
+    names.
+    """
+    if PY3:
+      import winreg as _winreg
+    else:
+      import _winreg
+
+    shell_folder_name = {
+        "CSIDL_APPDATA": "AppData",
+        "CSIDL_COMMON_APPDATA": "Common AppData",
+        "CSIDL_LOCAL_APPDATA": "Local AppData",
+    }[csidl_name]
+
+    key = _winreg.OpenKey(
+        _winreg.HKEY_CURRENT_USER,
+        r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+    )
+    dir, type = _winreg.QueryValueEx(key, shell_folder_name)
+    return dir
+
+
+def _get_win_folder_with_pywin32(csidl_name):
+    from win32com.shell import shellcon, shell
+    dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0)
+    # Try to make this a unicode path because SHGetFolderPath does
+    # not return unicode strings when there is unicode data in the
+    # path.
+    try:
+        dir = unicode(dir)
+
+        # Downgrade to short path name if have highbit chars. See
+        # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+        has_high_char = False
+        for c in dir:
+            if ord(c) > 255:
+                has_high_char = True
+                break
+        if has_high_char:
+            try:
+                import win32api
+                dir = win32api.GetShortPathName(dir)
+            except ImportError:
+                pass
+    except UnicodeError:
+        pass
+    return dir
+
+
+def _get_win_folder_with_ctypes(csidl_name):
+    import ctypes
+
+    csidl_const = {
+        "CSIDL_APPDATA": 26,
+        "CSIDL_COMMON_APPDATA": 35,
+        "CSIDL_LOCAL_APPDATA": 28,
+    }[csidl_name]
+
+    buf = ctypes.create_unicode_buffer(1024)
+    ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
+
+    # Downgrade to short path name if have highbit chars. See
+    # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+    has_high_char = False
+    for c in buf:
+        if ord(c) > 255:
+            has_high_char = True
+            break
+    if has_high_char:
+        buf2 = ctypes.create_unicode_buffer(1024)
+        if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024):
+            buf = buf2
+
+    return buf.value
+
+def _get_win_folder_with_jna(csidl_name):
+    import array
+    from com.sun import jna
+    from com.sun.jna.platform import win32
+
+    buf_size = win32.WinDef.MAX_PATH * 2
+    buf = array.zeros('c', buf_size)
+    shell = win32.Shell32.INSTANCE
+    shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf)
+    dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+    # Downgrade to short path name if have highbit chars. See
+    # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+    has_high_char = False
+    for c in dir:
+        if ord(c) > 255:
+            has_high_char = True
+            break
+    if has_high_char:
+        buf = array.zeros('c', buf_size)
+        kernel = win32.Kernel32.INSTANCE
+        if kernel.GetShortPathName(dir, buf, buf_size):
+            dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+    return dir
+
+if system == "win32":
+    try:
+        import win32com.shell
+        _get_win_folder = _get_win_folder_with_pywin32
+    except ImportError:
+        try:
+            from ctypes import windll
+            _get_win_folder = _get_win_folder_with_ctypes
+        except ImportError:
+            try:
+                import com.sun.jna
+                _get_win_folder = _get_win_folder_with_jna
+            except ImportError:
+                _get_win_folder = _get_win_folder_from_registry
+
+
+#---- self test code
+
+if __name__ == "__main__":
+    appname = "MyApp"
+    appauthor = "MyCompany"
+
+    props = ("user_data_dir",
+             "user_config_dir",
+             "user_cache_dir",
+             "user_state_dir",
+             "user_log_dir",
+             "site_data_dir",
+             "site_config_dir")
+
+    print("-- app dirs %s --" % __version__)
+
+    print("-- app dirs (with optional 'version')")
+    dirs = AppDirs(appname, appauthor, version="1.0")
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (without optional 'version')")
+    dirs = AppDirs(appname, appauthor)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (without optional 'appauthor')")
+    dirs = AppDirs(appname)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (with disabled 'appauthor')")
+    dirs = AppDirs(appname, appauthor=False)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))

lib/apscheduler/events.py

@@ -3,7 +3,7 @@ __all__ = ('EVENT_SCHEDULER_STARTED', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_SCHEDUL
            'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED',
            'EVENT_JOB_ADDED', 'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED',
            'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED', 'EVENT_JOB_SUBMITTED', 'EVENT_JOB_MAX_INSTANCES',
-           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent')
+           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent', 'JobSubmissionEvent')
 
 
 EVENT_SCHEDULER_STARTED = EVENT_SCHEDULER_START = 2 ** 0

lib/apscheduler/executors/asyncio.py

@@ -3,12 +3,11 @@ from __future__ import absolute_import
 import sys
 
 from apscheduler.executors.base import BaseExecutor, run_job
+from apscheduler.util import iscoroutinefunction_partial
 
 try:
-    from asyncio import iscoroutinefunction
     from apscheduler.executors.base_py3 import run_coroutine_job
 except ImportError:
-    from trollius import iscoroutinefunction
     run_coroutine_job = None
 
 
@@ -46,7 +45,7 @@ class AsyncIOExecutor(BaseExecutor):
             else:
                 self._run_job_success(job.id, events)
 
-        if iscoroutinefunction(job.func):
+        if iscoroutinefunction_partial(job.func):
             if run_coroutine_job is not None:
                 coro = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
                 f = self._eventloop.create_task(coro)

lib/apscheduler/executors/tornado.py

@@ -8,10 +8,10 @@ from tornado.gen import convert_yielded
 from apscheduler.executors.base import BaseExecutor, run_job
 
 try:
-    from inspect import iscoroutinefunction
     from apscheduler.executors.base_py3 import run_coroutine_job
+    from apscheduler.util import iscoroutinefunction_partial
 except ImportError:
-    def iscoroutinefunction(func):
+    def iscoroutinefunction_partial(func):
         return False
 
 
@@ -44,7 +44,7 @@ class TornadoExecutor(BaseExecutor):
             else:
                 self._run_job_success(job.id, events)
 
-        if iscoroutinefunction(job.func):
+        if iscoroutinefunction_partial(job.func):
             f = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
         else:
             f = self.executor.submit(run_job, job, job._jobstore_alias, run_times,

lib/apscheduler/job.py

@@ -1,4 +1,4 @@
-from collections import Iterable, Mapping
+from inspect import ismethod, isclass
 from uuid import uuid4
 
 import six
@@ -8,6 +8,11 @@ from apscheduler.util import (
     ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args,
     convert_to_datetime)
 
+try:
+    from collections.abc import Iterable, Mapping
+except ImportError:
+    from collections import Iterable, Mapping
+
 
 class Job(object):
     """
@@ -235,13 +240,20 @@ class Job(object):
                 'be determined. Consider giving a textual reference (module:function name) '
                 'instead.' % (self.func,))
 
+        # Instance methods cannot survive serialization as-is, so store the "self" argument
+        # explicitly
+        if ismethod(self.func) and not isclass(self.func.__self__):
+            args = (self.func.__self__,) + tuple(self.args)
+        else:
+            args = self.args
+
         return {
             'version': 1,
             'id': self.id,
             'func': self.func_ref,
             'trigger': self.trigger,
             'executor': self.executor,
-            'args': self.args,
+            'args': args,
             'kwargs': self.kwargs,
             'name': self.name,
             'misfire_grace_time': self.misfire_grace_time,

lib/apscheduler/jobstores/redis.py

@@ -14,7 +14,7 @@ except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from redis import StrictRedis
+    from redis import Redis
 except ImportError:  # pragma: nocover
     raise ImportError('RedisJobStore requires redis installed')
 
@@ -47,7 +47,7 @@ class RedisJobStore(BaseJobStore):
         self.pickle_protocol = pickle_protocol
         self.jobs_key = jobs_key
         self.run_times_key = run_times_key
-        self.redis = StrictRedis(db=int(db), **connect_args)
+        self.redis = Redis(db=int(db), **connect_args)
 
     def lookup_job(self, job_id):
         job_state = self.redis.hget(self.jobs_key, job_id)
@@ -81,7 +81,9 @@ class RedisJobStore(BaseJobStore):
             pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
                                                           self.pickle_protocol))
             if job.next_run_time:
-                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
+
             pipe.execute()
 
     def update_job(self, job):
@@ -92,9 +94,11 @@ class RedisJobStore(BaseJobStore):
             pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
                                                           self.pickle_protocol))
             if job.next_run_time:
-                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
             else:
                 pipe.zrem(self.run_times_key, job.id)
+
             pipe.execute()
 
     def remove_job(self, job_id):

lib/apscheduler/jobstores/rethinkdb.py

@@ -10,7 +10,7 @@ except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    import rethinkdb as r
+    from rethinkdb import RethinkDB
 except ImportError:  # pragma: nocover
     raise ImportError('RethinkDBJobStore requires rethinkdb installed')
 
@@ -40,10 +40,12 @@ class RethinkDBJobStore(BaseJobStore):
             raise ValueError('The "table" parameter must not be empty')
 
         self.database = database
-        self.table = table
+        self.table_name = table
+        self.table = None
         self.client = client
         self.pickle_protocol = pickle_protocol
         self.connect_args = connect_args
+        self.r = RethinkDB()
         self.conn = None
 
     def start(self, scheduler, alias):
@@ -52,31 +54,31 @@ class RethinkDBJobStore(BaseJobStore):
         if self.client:
             self.conn = maybe_ref(self.client)
         else:
-            self.conn = r.connect(db=self.database, **self.connect_args)
+            self.conn = self.r.connect(db=self.database, **self.connect_args)
 
-        if self.database not in r.db_list().run(self.conn):
-            r.db_create(self.database).run(self.conn)
+        if self.database not in self.r.db_list().run(self.conn):
+            self.r.db_create(self.database).run(self.conn)
 
-        if self.table not in r.table_list().run(self.conn):
-            r.table_create(self.table).run(self.conn)
+        if self.table_name not in self.r.table_list().run(self.conn):
+            self.r.table_create(self.table_name).run(self.conn)
 
-        if 'next_run_time' not in r.table(self.table).index_list().run(self.conn):
-            r.table(self.table).index_create('next_run_time').run(self.conn)
+        if 'next_run_time' not in self.r.table(self.table_name).index_list().run(self.conn):
+            self.r.table(self.table_name).index_create('next_run_time').run(self.conn)
 
-        self.table = r.db(self.database).table(self.table)
+        self.table = self.r.db(self.database).table(self.table_name)
 
     def lookup_job(self, job_id):
         results = list(self.table.get_all(job_id).pluck('job_state').run(self.conn))
         return self._reconstitute_job(results[0]['job_state']) if results else None
 
     def get_due_jobs(self, now):
-        return self._get_jobs(r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
+        return self._get_jobs(self.r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
 
     def get_next_run_time(self):
         results = list(
             self.table
-            .filter(r.row['next_run_time'] != None)  # flake8: noqa
-            .order_by(r.asc('next_run_time'))
+            .filter(self.r.row['next_run_time'] != None)  # noqa
+            .order_by(self.r.asc('next_run_time'))
             .map(lambda x: x['next_run_time'])
             .limit(1)
             .run(self.conn)
@@ -92,7 +94,7 @@ class RethinkDBJobStore(BaseJobStore):
         job_dict = {
             'id': job.id,
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
-            'job_state': r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
         }
         results = self.table.insert(job_dict).run(self.conn)
         if results['errors'] > 0:
@@ -101,7 +103,7 @@ class RethinkDBJobStore(BaseJobStore):
     def update_job(self, job):
         changes = {
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
-            'job_state': r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
         }
         results = self.table.get_all(job.id).update(changes).run(self.conn)
         skipped = False in map(lambda x: results[x] == 0, results.keys())
@@ -130,20 +132,20 @@ class RethinkDBJobStore(BaseJobStore):
     def _get_jobs(self, predicate=None):
         jobs = []
         failed_job_ids = []
-        query = (self.table.filter(r.row['next_run_time'] != None).filter(predicate) if
-                 predicate else self.table)
+        query = (self.table.filter(self.r.row['next_run_time'] != None).filter(predicate)  # noqa
+                 if predicate else self.table)
         query = query.order_by('next_run_time', 'id').pluck('id', 'job_state')
 
         for document in query.run(self.conn):
             try:
                 jobs.append(self._reconstitute_job(document['job_state']))
-            except:
+            except Exception:
                 self._logger.exception('Unable to restore job "%s" -- removing it', document['id'])
                 failed_job_ids.append(document['id'])
 
         # Remove all the jobs we failed to restore
         if failed_job_ids:
-            r.expr(failed_job_ids).for_each(
+            self.r.expr(failed_job_ids).for_each(
                 lambda job_id: self.table.get_all(job_id).delete()).run(self.conn)
 
         return jobs

lib/apscheduler/jobstores/sqlalchemy.py

@@ -106,7 +106,7 @@ class SQLAlchemyJobStore(BaseJobStore):
         }).where(self.jobs_t.c.id == job.id)
         result = self.engine.execute(update)
         if result.rowcount == 0:
-            raise JobLookupError(id)
+            raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
         delete = self.jobs_t.delete().where(self.jobs_t.c.id == job_id)

lib/apscheduler/schedulers/base.py

@@ -1,7 +1,6 @@
 from __future__ import print_function
 
 from abc import ABCMeta, abstractmethod
-from collections import MutableMapping
 from threading import RLock
 from datetime import datetime, timedelta
 from logging import getLogger
@@ -19,13 +18,19 @@ from apscheduler.jobstores.base import ConflictingIdError, JobLookupError, BaseJ
 from apscheduler.jobstores.memory import MemoryJobStore
 from apscheduler.job import Job
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import asbool, asint, astimezone, maybe_ref, timedelta_seconds, undefined
+from apscheduler.util import (
+    asbool, asint, astimezone, maybe_ref, timedelta_seconds, undefined, TIMEOUT_MAX)
 from apscheduler.events import (
     SchedulerEvent, JobEvent, JobSubmissionEvent, EVENT_SCHEDULER_START, EVENT_SCHEDULER_SHUTDOWN,
     EVENT_JOBSTORE_ADDED, EVENT_JOBSTORE_REMOVED, EVENT_ALL, EVENT_JOB_MODIFIED, EVENT_JOB_REMOVED,
     EVENT_JOB_ADDED, EVENT_EXECUTOR_ADDED, EVENT_EXECUTOR_REMOVED, EVENT_ALL_JOBS_REMOVED,
     EVENT_JOB_SUBMITTED, EVENT_JOB_MAX_INSTANCES, EVENT_SCHEDULER_RESUMED, EVENT_SCHEDULER_PAUSED)
 
+try:
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+
 #: constant indicating a scheduler's stopped state
 STATE_STOPPED = 0
 #: constant indicating a scheduler's running state (started and processing jobs)
@@ -126,11 +131,14 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         :param bool paused: if ``True``, don't start job processing until :meth:`resume` is called
         :raises SchedulerAlreadyRunningError: if the scheduler is already running
+        :raises RuntimeError: if running under uWSGI with threads disabled
 
         """
         if self.state != STATE_STOPPED:
             raise SchedulerAlreadyRunningError
 
+        self._check_uwsgi()
+
         with self._executors_lock:
             # Create a default executor if nothing else is configured
             if 'default' not in self._executors:
@@ -177,12 +185,13 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         self.state = STATE_STOPPED
 
-        with self._jobstores_lock, self._executors_lock:
-            # Shut down all executors
+        # Shut down all executors
+        with self._executors_lock:
             for executor in six.itervalues(self._executors):
                 executor.shutdown(wait)
 
-            # Shut down all job stores
+        # Shut down all job stores
+        with self._jobstores_lock:
             for jobstore in six.itervalues(self._jobstores):
                 jobstore.shutdown()
 
@@ -546,7 +555,7 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         """
         if pending is not None:
             warnings.warn('The "pending" option is deprecated -- get_jobs() always returns '
-                          'pending jobs if the scheduler has been started and scheduled jobs '
+                          'scheduled jobs if the scheduler has been started and pending jobs '
                           'otherwise', DeprecationWarning)
 
         with self._jobstores_lock:
@@ -589,14 +598,13 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         """
         jobstore_alias = None
         with self._jobstores_lock:
+            # Check if the job is among the pending jobs
             if self.state == STATE_STOPPED:
-                # Check if the job is among the pending jobs
-                if self.state == STATE_STOPPED:
-                    for i, (job, alias, replace_existing) in enumerate(self._pending_jobs):
-                        if job.id == job_id and jobstore in (None, alias):
-                            del self._pending_jobs[i]
-                            jobstore_alias = alias
-                            break
+                for i, (job, alias, replace_existing) in enumerate(self._pending_jobs):
+                    if job.id == job_id and jobstore in (None, alias):
+                        del self._pending_jobs[i]
+                        jobstore_alias = alias
+                        break
             else:
                 # Otherwise, try to remove it from each store until it succeeds or we run out of
                 # stores to check
@@ -824,6 +832,14 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
                 except BaseException:
                     self._logger.exception('Error notifying listener')
 
+    def _check_uwsgi(self):
+        """Check if we're running under uWSGI with threads disabled."""
+        uwsgi_module = sys.modules.get('uwsgi')
+        if not getattr(uwsgi_module, 'has_threads', True):
+            raise RuntimeError('The scheduler seems to be running under uWSGI, but threads have '
+                               'been disabled. You must run uWSGI with the --enable-threads '
+                               'option for the scheduler to work.')
+
     def _real_add_job(self, job, jobstore_alias, replace_existing):
         """
         :param Job job: the job to add
@@ -999,7 +1015,7 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
             wait_seconds = None
             self._logger.debug('No jobs; waiting until a job is added')
         else:
-            wait_seconds = max(timedelta_seconds(next_wakeup_time - now), 0)
+            wait_seconds = min(max(timedelta_seconds(next_wakeup_time - now), 0), TIMEOUT_MAX)
             self._logger.debug('Next wakeup is due at %s (in %f seconds)', next_wakeup_time,
                                wait_seconds)
 

lib/apscheduler/schedulers/qt.py

@@ -9,7 +9,7 @@ except (ImportError, RuntimeError):  # pragma: nocover
         from PyQt4.QtCore import QObject, QTimer
     except ImportError:
         try:
-            from PySide.QtCore import QObject, QTimer  # flake8: noqa
+            from PySide.QtCore import QObject, QTimer  # noqa
         except ImportError:
             raise ImportError('QtScheduler requires either PyQt5, PyQt4 or PySide installed')
 
@@ -26,7 +26,8 @@ class QtScheduler(BaseScheduler):
     def _start_timer(self, wait_seconds):
         self._stop_timer()
         if wait_seconds is not None:
-            self._timer = QTimer.singleShot(wait_seconds * 1000, self._process_jobs)
+            wait_time = min(wait_seconds * 1000, 2147483647)
+            self._timer = QTimer.singleShot(wait_time, self._process_jobs)
 
     def _stop_timer(self):
         if self._timer:

lib/apscheduler/triggers/cron/__init__.py

@@ -192,9 +192,8 @@ class CronTrigger(BaseTrigger):
                 return None
 
         if fieldnum >= 0:
-            if self.jitter is not None:
-                next_date = self._apply_jitter(next_date, self.jitter, now)
-            return next_date
+            next_date = self._apply_jitter(next_date, self.jitter, now)
+            return min(next_date, self.end_date) if self.end_date else next_date
 
     def __getstate__(self):
         return {

lib/apscheduler/triggers/cron/expressions.py

@@ -9,7 +9,7 @@ __all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression',
            'WeekdayPositionExpression', 'LastDayOfMonthExpression')
 
 
-WEEKDAYS = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']
+WEEKDAYS = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
 MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
 
 

lib/apscheduler/triggers/cron/fields.py

@@ -104,7 +104,7 @@ class DayOfWeekField(BaseField):
     COMPILERS = BaseField.COMPILERS + [WeekdayRangeExpression]
 
     def get_value(self, dateval):
-        return dateval.isoweekday() % 7
+        return dateval.weekday()
 
 
 class MonthField(BaseField):

lib/apscheduler/util.py

@@ -1,12 +1,14 @@
 """This module contains several handy functions primarily meant for internal use."""
 
 from __future__ import division
+
 from datetime import date, datetime, time, timedelta, tzinfo
 from calendar import timegm
-import re
 from functools import partial
+from inspect import isclass, ismethod
+import re
 
-from pytz import timezone, utc
+from pytz import timezone, utc, FixedOffset
 import six
 
 try:
@@ -19,9 +21,19 @@ try:
 except ImportError:
     TIMEOUT_MAX = 4294967  # Maximum value accepted by Event.wait() on Windows
 
+try:
+    from asyncio import iscoroutinefunction
+except ImportError:
+    try:
+        from trollius import iscoroutinefunction
+    except ImportError:
+        def iscoroutinefunction(func):
+            return False
+
 __all__ = ('asint', 'asbool', 'astimezone', 'convert_to_datetime', 'datetime_to_utc_timestamp',
            'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name',
-           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args')
+           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args',
+           'TIMEOUT_MAX')
 
 
 class _Undefined(object):
@@ -92,8 +104,9 @@ def astimezone(obj):
 
 _DATE_REGEX = re.compile(
     r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})'
-    r'(?: (?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
-    r'(?:\.(?P<microsecond>\d{1,6}))?)?')
+    r'(?:[ T](?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
+    r'(?:\.(?P<microsecond>\d{1,6}))?'
+    r'(?P<timezone>Z|[+-]\d\d:\d\d)?)?$')
 
 
 def convert_to_datetime(input, tz, arg_name):
@@ -105,7 +118,9 @@ def convert_to_datetime(input, tz, arg_name):
     If the input is a string, it is parsed as a datetime with the given timezone.
 
     Date strings are accepted in three different forms: date only (Y-m-d), date with time
-    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro).
+    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro). Additionally you can
+    override the time zone by giving a specific offset in the format specified by ISO 8601:
+    Z (UTC), +HH:MM or -HH:MM.
 
     :param str|datetime input: the datetime or string to convert to a timezone aware datetime
     :param datetime.tzinfo tz: timezone to interpret ``input`` in
@@ -123,8 +138,17 @@ def convert_to_datetime(input, tz, arg_name):
         m = _DATE_REGEX.match(input)
         if not m:
             raise ValueError('Invalid date string')
-        values = [(k, int(v or 0)) for k, v in m.groupdict().items()]
-        values = dict(values)
+
+        values = m.groupdict()
+        tzname = values.pop('timezone')
+        if tzname == 'Z':
+            tz = utc
+        elif tzname:
+            hours, minutes = (int(x) for x in tzname[1:].split(':'))
+            sign = 1 if tzname[0] == '+' else -1
+            tz = FixedOffset(sign * (hours * 60 + minutes))
+
+        values = {k: int(v or 0) for k, v in values.items()}
         datetime_ = datetime(**values)
     else:
         raise TypeError('Unsupported type for %s: %s' % (arg_name, input.__class__.__name__))
@@ -210,7 +234,7 @@ def get_callable_name(func):
     # class methods, bound and unbound methods
     f_self = getattr(func, '__self__', None) or getattr(func, 'im_self', None)
     if f_self and hasattr(func, '__name__'):
-        f_class = f_self if isinstance(f_self, type) else f_self.__class__
+        f_class = f_self if isclass(f_self) else f_self.__class__
     else:
         f_class = getattr(func, 'im_class', None)
 
@@ -248,7 +272,18 @@ def obj_to_ref(obj):
     if '<locals>' in name:
         raise ValueError('Cannot create a reference to a nested function')
 
-    return '%s:%s' % (obj.__module__, name)
+    if ismethod(obj):
+        if hasattr(obj, 'im_self') and obj.im_self:
+            # bound method
+            module = obj.im_self.__module__
+        elif hasattr(obj, 'im_class') and obj.im_class:
+            # unbound method
+            module = obj.im_class.__module__
+        else:
+            module = obj.__module__
+    else:
+        module = obj.__module__
+    return '%s:%s' % (module, name)
 
 
 def ref_to_obj(ref):
@@ -383,3 +418,12 @@ def check_callable_args(func, args, kwargs):
         raise ValueError(
             'The target callable does not accept the following keyword arguments: %s' %
             ', '.join(unmatched_kwargs))
+
+
+def iscoroutinefunction_partial(f):
+    while isinstance(f, partial):
+        f = f.func
+
+    # The asyncio version of iscoroutinefunction includes testing for @coroutine
+    # decorations vs. the inspect version which does not.
+    return iscoroutinefunction(f)

lib/argparse.py

@@ -1,4 +1,5 @@
 # Author: Steven J. Bethard <steven.bethard@gmail.com>.
+# Maintainer: Thomas Waldmann <tw@waldmann-edv.de>
 
 """Command-line parsing library
 
@@ -61,7 +62,12 @@ considered public as object names -- the API of the formatter objects is
 still considered an implementation detail.)
 """
 
-__version__ = '1.1'
+__version__ = '1.4.0'  # we use our own version number independant of the
+                       # one in stdlib and we release this on pypi.
+
+__external_lib__ = True  # to make sure the tests really test THIS lib,
+                         # not the builtin one in Python stdlib
+
 __all__ = [
     'ArgumentParser',
     'ArgumentError',
@@ -71,7 +77,6 @@ __all__ = [
     'ArgumentDefaultsHelpFormatter',
     'RawDescriptionHelpFormatter',
     'RawTextHelpFormatter',
-    'MetavarTypeHelpFormatter',
     'Namespace',
     'Action',
     'ONE_OR_MORE',
@@ -83,14 +88,35 @@ __all__ = [
 ]
 
 
-import collections as _collections
 import copy as _copy
 import os as _os
 import re as _re
 import sys as _sys
 import textwrap as _textwrap
 
-from gettext import gettext as _, ngettext
+from gettext import gettext as _
+
+try:
+    set
+except NameError:
+    # for python < 2.4 compatibility (sets module is there since 2.3):
+    from sets import Set as set
+
+try:
+    basestring
+except NameError:
+    basestring = str
+
+try:
+    sorted
+except NameError:
+    # for python < 2.4 compatibility:
+    def sorted(iterable, reverse=False):
+        result = list(iterable)
+        result.sort()
+        if reverse:
+            result.reverse()
+        return result
 
 
 def _callable(obj):
@@ -424,8 +450,7 @@ class HelpFormatter(object):
 
             # produce all arg strings
             elif not action.option_strings:
-                default = self._get_default_metavar_for_positional(action)
-                part = self._format_args(action, default)
+                part = self._format_args(action, action.dest)
 
                 # if it's in a group, strip the outer []
                 if action in group_actions:
@@ -447,7 +472,7 @@ class HelpFormatter(object):
                 # if the Optional takes a value, format is:
                 #    -s ARGS or --long ARGS
                 else:
-                    default = self._get_default_metavar_for_optional(action)
+                    default = action.dest.upper()
                     args_string = self._format_args(action, default)
                     part = '%s %s' % (option_string, args_string)
 
@@ -533,8 +558,7 @@ class HelpFormatter(object):
 
     def _format_action_invocation(self, action):
         if not action.option_strings:
-            default = self._get_default_metavar_for_positional(action)
-            metavar, = self._metavar_formatter(action, default)(1)
+            metavar, = self._metavar_formatter(action, action.dest)(1)
             return metavar
 
         else:
@@ -548,7 +572,7 @@ class HelpFormatter(object):
             # if the Optional takes a value, format is:
             #    -s ARGS, --long ARGS
             else:
-                default = self._get_default_metavar_for_optional(action)
+                default = action.dest.upper()
                 args_string = self._format_args(action, default)
                 for option_string in action.option_strings:
                     parts.append('%s %s' % (option_string, args_string))
@@ -626,12 +650,6 @@ class HelpFormatter(object):
     def _get_help_string(self, action):
         return action.help
 
-    def _get_default_metavar_for_optional(self, action):
-        return action.dest.upper()
-
-    def _get_default_metavar_for_positional(self, action):
-        return action.dest
-
 
 class RawDescriptionHelpFormatter(HelpFormatter):
     """Help message formatter which retains any formatting in descriptions.
@@ -672,22 +690,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
         return help
 
 
-class MetavarTypeHelpFormatter(HelpFormatter):
-    """Help message formatter which uses the argument 'type' as the default
-    metavar value (instead of the argument 'dest')
-
-    Only the name of this class is considered a public API. All the methods
-    provided by the class are considered an implementation detail.
-    """
-
-    def _get_default_metavar_for_optional(self, action):
-        return action.type.__name__
-
-    def _get_default_metavar_for_positional(self, action):
-        return action.type.__name__
-
-
-
 # =====================
 # Options and Arguments
 # =====================
@@ -1055,7 +1057,7 @@ class _SubParsersAction(Action):
                 metavar += ' (%s)' % ', '.join(aliases)
             sup = super(_SubParsersAction._ChoicesPseudoAction, self)
             sup.__init__(option_strings=[], dest=dest, help=help,
-                         metavar=metavar)
+                        metavar=metavar)
 
     def __init__(self,
                  option_strings,
@@ -1067,7 +1069,7 @@ class _SubParsersAction(Action):
 
         self._prog_prefix = prog
         self._parser_class = parser_class
-        self._name_parser_map = _collections.OrderedDict()
+        self._name_parser_map = {}
         self._choices_actions = []
 
         super(_SubParsersAction, self).__init__(
@@ -1116,9 +1118,8 @@ class _SubParsersAction(Action):
         try:
             parser = self._name_parser_map[parser_name]
         except KeyError:
-            args = {'parser_name': parser_name,
-                    'choices': ', '.join(self._name_parser_map)}
-            msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args
+            tup = parser_name, ', '.join(self._name_parser_map)
+            msg = _('unknown parser %r (choices: %s)' % tup)
             raise ArgumentError(self, msg)
 
         # parse all the remaining options into the namespace
@@ -1147,7 +1148,7 @@ class FileType(object):
             the builtin open() function.
     """
 
-    def __init__(self, mode='r', bufsize=-1):
+    def __init__(self, mode='r', bufsize=None):
         self._mode = mode
         self._bufsize = bufsize
 
@@ -1159,19 +1160,23 @@ class FileType(object):
             elif 'w' in self._mode:
                 return _sys.stdout
             else:
-                msg = _('argument "-" with mode %r') % self._mode
+                msg = _('argument "-" with mode %r' % self._mode)
                 raise ValueError(msg)
 
-        # all other arguments are used as file names
         try:
-            return open(string, self._mode, self._bufsize)
-        except IOError as e:
+            # all other arguments are used as file names
+            if self._bufsize:
+                return open(string, self._mode, self._bufsize)
+            else:
+                return open(string, self._mode)
+        except IOError:
+            err = _sys.exc_info()[1]
             message = _("can't open '%s': %s")
-            raise ArgumentTypeError(message % (string, e))
+            raise ArgumentTypeError(message % (string, err))
 
     def __repr__(self):
-        args = self._mode, self._bufsize
-        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
+        args = [self._mode, self._bufsize]
+        args_str = ', '.join([repr(arg) for arg in args if arg is not None])
         return '%s(%s)' % (type(self).__name__, args_str)
 
 # ===========================
@@ -1189,6 +1194,8 @@ class Namespace(_AttributeHolder):
         for name in kwargs:
             setattr(self, name, kwargs[name])
 
+    __hash__ = None
+
     def __eq__(self, other):
         return vars(self) == vars(other)
 
@@ -1312,20 +1319,13 @@ class _ActionsContainer(object):
         # create the action object, and add it to the parser
         action_class = self._pop_action_class(kwargs)
         if not _callable(action_class):
-            raise ValueError('unknown action "%s"' % (action_class,))
+            raise ValueError('unknown action "%s"' % action_class)
         action = action_class(**kwargs)
 
         # raise an error if the action type is not callable
         type_func = self._registry_get('type', action.type, action.type)
         if not _callable(type_func):
-            raise ValueError('%r is not callable' % (type_func,))
-
-        # raise an error if the metavar does not match the type
-        if hasattr(self, "_get_formatter"):
-            try:
-                self._get_formatter()._format_args(action, None)
-            except TypeError:
-                raise ValueError("length of metavar tuple does not match nargs")
+            raise ValueError('%r is not callable' % type_func)
 
         return self._add_action(action)
 
@@ -1426,11 +1426,10 @@ class _ActionsContainer(object):
         for option_string in args:
             # error on strings that don't start with an appropriate prefix
             if not option_string[0] in self.prefix_chars:
-                args = {'option': option_string,
-                        'prefix_chars': self.prefix_chars}
-                msg = _('invalid option string %(option)r: '
-                        'must start with a character %(prefix_chars)r')
-                raise ValueError(msg % args)
+                msg = _('invalid option string %r: '
+                        'must start with a character %r')
+                tup = option_string, self.prefix_chars
+                raise ValueError(msg % tup)
 
             # strings starting with two prefix characters are long options
             option_strings.append(option_string)
@@ -1483,9 +1482,7 @@ class _ActionsContainer(object):
             conflict_handler(action, confl_optionals)
 
     def _handle_conflict_error(self, action, conflicting_actions):
-        message = ngettext('conflicting option string: %s',
-                           'conflicting option strings: %s',
-                           len(conflicting_actions))
+        message = _('conflicting option string(s): %s')
         conflict_string = ', '.join([option_string
                                      for option_string, action
                                      in conflicting_actions])
@@ -1528,7 +1525,6 @@ class _ArgumentGroup(_ActionsContainer):
         self._defaults = container._defaults
         self._has_negative_number_optionals = \
             container._has_negative_number_optionals
-        self._mutually_exclusive_groups = container._mutually_exclusive_groups
 
     def _add_action(self, action):
         action = super(_ArgumentGroup, self)._add_action(action)
@@ -1630,7 +1626,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
 
         # add help and version arguments if necessary
         # (using explicit default to override global argument_default)
-        default_prefix = '-' if '-' in prefix_chars else prefix_chars[0]
+        if '-' in prefix_chars:
+            default_prefix = '-'
+        else:
+            default_prefix = prefix_chars[0]
         if self.add_help:
             self.add_argument(
                 default_prefix+'h', default_prefix*2+'help',
@@ -1743,10 +1742,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
             if action.dest is not SUPPRESS:
                 if not hasattr(namespace, action.dest):
                     if action.default is not SUPPRESS:
-                        default = action.default
-                        if isinstance(action.default, str):
-                            default = self._get_value(action, default)
-                        setattr(namespace, action.dest, default)
+                        setattr(namespace, action.dest, action.default)
 
         # add any parser defaults that aren't present
         for dest in self._defaults:
@@ -1969,12 +1965,28 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         # if we didn't consume all the argument strings, there were extras
         extras.extend(arg_strings[stop_index:])
 
-        # make sure all required actions were present
-        required_actions = [_get_action_name(action) for action in self._actions
-                            if action.required and action not in seen_actions]
-        if required_actions:
-            self.error(_('the following arguments are required: %s') %
-                       ', '.join(required_actions))
+        # if we didn't use all the Positional objects, there were too few
+        # arg strings supplied.
+        if positionals:
+            self.error(_('too few arguments'))
+
+        # make sure all required actions were present, and convert defaults.
+        for action in self._actions:
+            if action not in seen_actions:
+                if action.required:
+                    name = _get_action_name(action)
+                    self.error(_('argument %s is required') % name)
+                else:
+                    # Convert action default now instead of doing it before
+                    # parsing arguments to avoid calling convert functions
+                    # twice (which may fail) if the argument was given, but
+                    # only if it was defined already in the namespace
+                    if (action.default is not None and
+                            isinstance(action.default, basestring) and
+                            hasattr(namespace, action.dest) and
+                            action.default is getattr(namespace, action.dest)):
+                        setattr(namespace, action.dest,
+                                self._get_value(action, action.default))
 
         # make sure all required groups had one option present
         for group in self._mutually_exclusive_groups:
@@ -2038,9 +2050,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 OPTIONAL: _('expected at most one argument'),
                 ONE_OR_MORE: _('expected at least one argument'),
             }
-            default = ngettext('expected %s argument',
-                               'expected %s arguments',
-                               action.nargs) % action.nargs
+            default = _('expected %s argument(s)') % action.nargs
             msg = nargs_errors.get(action.nargs, default)
             raise ArgumentError(action, msg)
 
@@ -2096,9 +2106,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         if len(option_tuples) > 1:
             options = ', '.join([option_string
                 for action, option_string, explicit_arg in option_tuples])
-            args = {'option': arg_string, 'matches': options}
-            msg = _('ambiguous option: %(option)s could match %(matches)s')
-            self.error(msg % args)
+            tup = arg_string, options
+            self.error(_('ambiguous option: %s could match %s') % tup)
 
         # if exactly one action matched, this segmentation is good,
         # so return the parsed action
@@ -2220,7 +2229,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 value = action.const
             else:
                 value = action.default
-            if isinstance(value, str):
+            if isinstance(value, basestring):
                 value = self._get_value(action, value)
                 self._check_value(action, value)
 
@@ -2277,9 +2286,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         # TypeErrors or ValueErrors also indicate errors
         except (TypeError, ValueError):
             name = getattr(action.type, '__name__', repr(action.type))
-            args = {'type': name, 'value': arg_string}
-            msg = _('invalid %(type)s value: %(value)r')
-            raise ArgumentError(action, msg % args)
+            msg = _('invalid %s value: %r')
+            raise ArgumentError(action, msg % (name, arg_string))
 
         # return the converted value
         return result
@@ -2287,10 +2295,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
     def _check_value(self, action, value):
         # converted value must be one of the choices (if specified)
         if action.choices is not None and value not in action.choices:
-            args = {'value': value,
-                    'choices': ', '.join(map(repr, action.choices))}
-            msg = _('invalid choice: %(value)r (choose from %(choices)s)')
-            raise ArgumentError(action, msg % args)
+            tup = value, ', '.join(map(repr, action.choices))
+            msg = _('invalid choice: %r (choose from %s)') % tup
+            raise ArgumentError(action, msg)
 
     # =======================
     # Help-formatting methods
@@ -2382,5 +2389,4 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         should either exit or raise an exception.
         """
         self.print_usage(_sys.stderr)
-        args = {'prog': self.prog, 'message': message}
-        self.exit(2, _('%(prog)s: error: %(message)s\n') % args)
+        self.exit(2, _('%s: error: %s\n') % (self.prog, message))

lib/backports/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

lib/backports/functools_lru_cache.py

@@ -0,0 +1,196 @@
+from __future__ import absolute_import
+
+import functools
+from collections import namedtuple
+from threading import RLock
+
+_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
+
+
+@functools.wraps(functools.update_wrapper)
+def update_wrapper(
+    wrapper,
+    wrapped,
+    assigned=functools.WRAPPER_ASSIGNMENTS,
+    updated=functools.WRAPPER_UPDATES,
+):
+    """
+    Patch two bugs in functools.update_wrapper.
+    """
+    # workaround for http://bugs.python.org/issue3445
+    assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr))
+    wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated)
+    # workaround for https://bugs.python.org/issue17482
+    wrapper.__wrapped__ = wrapped
+    return wrapper
+
+
+class _HashedSeq(list):
+    __slots__ = 'hashvalue'
+
+    def __init__(self, tup, hash=hash):
+        self[:] = tup
+        self.hashvalue = hash(tup)
+
+    def __hash__(self):
+        return self.hashvalue
+
+
+def _make_key(
+    args,
+    kwds,
+    typed,
+    kwd_mark=(object(),),
+    fasttypes=set([int, str, frozenset, type(None)]),
+    sorted=sorted,
+    tuple=tuple,
+    type=type,
+    len=len,
+):
+    'Make a cache key from optionally typed positional and keyword arguments'
+    key = args
+    if kwds:
+        sorted_items = sorted(kwds.items())
+        key += kwd_mark
+        for item in sorted_items:
+            key += item
+    if typed:
+        key += tuple(type(v) for v in args)
+        if kwds:
+            key += tuple(type(v) for k, v in sorted_items)
+    elif len(key) == 1 and type(key[0]) in fasttypes:
+        return key[0]
+    return _HashedSeq(key)
+
+
+def lru_cache(maxsize=100, typed=False):
+    """Least-recently-used cache decorator.
+
+    If *maxsize* is set to None, the LRU features are disabled and the cache
+    can grow without bound.
+
+    If *typed* is True, arguments of different types will be cached separately.
+    For example, f(3.0) and f(3) will be treated as distinct calls with
+    distinct results.
+
+    Arguments to the cached function must be hashable.
+
+    View the cache statistics named tuple (hits, misses, maxsize, currsize) with
+    f.cache_info().  Clear the cache and statistics with f.cache_clear().
+    Access the underlying function with f.__wrapped__.
+
+    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
+
+    """
+
+    # Users should only access the lru_cache through its public API:
+    #       cache_info, cache_clear, and f.__wrapped__
+    # The internals of the lru_cache are encapsulated for thread safety and
+    # to allow the implementation to change (including a possible C version).
+
+    def decorating_function(user_function):
+
+        cache = dict()
+        stats = [0, 0]  # make statistics updateable non-locally
+        HITS, MISSES = 0, 1  # names for the stats fields
+        make_key = _make_key
+        cache_get = cache.get  # bound method to lookup key or return None
+        _len = len  # localize the global len() function
+        lock = RLock()  # because linkedlist updates aren't threadsafe
+        root = []  # root of the circular doubly linked list
+        root[:] = [root, root, None, None]  # initialize by pointing to self
+        nonlocal_root = [root]  # make updateable non-locally
+        PREV, NEXT, KEY, RESULT = 0, 1, 2, 3  # names for the link fields
+
+        if maxsize == 0:
+
+            def wrapper(*args, **kwds):
+                # no caching, just do a statistics update after a successful call
+                result = user_function(*args, **kwds)
+                stats[MISSES] += 1
+                return result
+
+        elif maxsize is None:
+
+            def wrapper(*args, **kwds):
+                # simple caching without ordering or size limit
+                key = make_key(args, kwds, typed)
+                result = cache_get(
+                    key, root
+                )  # root used here as a unique not-found sentinel
+                if result is not root:
+                    stats[HITS] += 1
+                    return result
+                result = user_function(*args, **kwds)
+                cache[key] = result
+                stats[MISSES] += 1
+                return result
+
+        else:
+
+            def wrapper(*args, **kwds):
+                # size limited caching that tracks accesses by recency
+                key = make_key(args, kwds, typed) if kwds or typed else args
+                with lock:
+                    link = cache_get(key)
+                    if link is not None:
+                        # record recent use of the key by moving it
+                        # to the front of the list
+                        root, = nonlocal_root
+                        link_prev, link_next, key, result = link
+                        link_prev[NEXT] = link_next
+                        link_next[PREV] = link_prev
+                        last = root[PREV]
+                        last[NEXT] = root[PREV] = link
+                        link[PREV] = last
+                        link[NEXT] = root
+                        stats[HITS] += 1
+                        return result
+                result = user_function(*args, **kwds)
+                with lock:
+                    root, = nonlocal_root
+                    if key in cache:
+                        # getting here means that this same key was added to the
+                        # cache while the lock was released.  since the link
+                        # update is already done, we need only return the
+                        # computed result and update the count of misses.
+                        pass
+                    elif _len(cache) >= maxsize:
+                        # use the old root to store the new key and result
+                        oldroot = root
+                        oldroot[KEY] = key
+                        oldroot[RESULT] = result
+                        # empty the oldest link and make it the new root
+                        root = nonlocal_root[0] = oldroot[NEXT]
+                        oldkey = root[KEY]
+                        root[KEY] = root[RESULT] = None
+                        # now update the cache dictionary for the new links
+                        del cache[oldkey]
+                        cache[key] = oldroot
+                    else:
+                        # put result in a new link at the front of the list
+                        last = root[PREV]
+                        link = [last, root, key, result]
+                        last[NEXT] = root[PREV] = cache[key] = link
+                    stats[MISSES] += 1
+                return result
+
+        def cache_info():
+            """Report cache statistics"""
+            with lock:
+                return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache))
+
+        def cache_clear():
+            """Clear the cache and cache statistics"""
+            with lock:
+                cache.clear()
+                root = nonlocal_root[0]
+                root[:] = [root, root, None, None]
+                stats[:] = [0, 0]
+
+        wrapper.__wrapped__ = user_function
+        wrapper.cache_info = cache_info
+        wrapper.cache_clear = cache_clear
+        return update_wrapper(wrapper, user_function)
+
+    return decorating_function

lib/bs4/__init__.py

@@ -5,26 +5,30 @@ http://www.crummy.com/software/BeautifulSoup/
 
 Beautiful Soup uses a pluggable XML or HTML parser to parse a
 (possibly invalid) document into a tree representation. Beautiful Soup
-provides provides methods and Pythonic idioms that make it easy to
-navigate, search, and modify the parse tree.
+provides methods and Pythonic idioms that make it easy to navigate,
+search, and modify the parse tree.
 
-Beautiful Soup works with Python 2.6 and up. It works better if lxml
+Beautiful Soup works with Python 2.7 and up. It works better if lxml
 and/or html5lib is installed.
 
 For more than you ever wanted to know about Beautiful Soup, see the
 documentation:
 http://www.crummy.com/software/BeautifulSoup/bs4/doc/
+
 """
 
 __author__ = "Leonard Richardson (leonardr@segfault.org)"
-__version__ = "4.3.2"
-__copyright__ = "Copyright (c) 2004-2013 Leonard Richardson"
+__version__ = "4.8.1"
+__copyright__ = "Copyright (c) 2004-2019 Leonard Richardson"
+# Use of this source code is governed by the MIT license.
 __license__ = "MIT"
 
 __all__ = ['BeautifulSoup']
 
 import os
 import re
+import sys
+import traceback
 import warnings
 
 from .builder import builder_registry, ParserRejectedMarkup
@@ -45,7 +49,7 @@ from .element import (
 
 # The very first thing we do is give a useful error if someone is
 # running this code under Python 3 without converting it.
-syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
+'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'!='You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
 
 class BeautifulSoup(Tag):
     """
@@ -59,7 +63,7 @@ class BeautifulSoup(Tag):
       handle_starttag(name, attrs) # See note about return value
       handle_endtag(name)
       handle_data(data) # Appends to the current data node
-      endData(containerClass=NavigableString) # Ends the current data node
+      endData(containerClass) # Ends the current data node
 
     No matter how complicated the underlying parser is, you should be
     able to build a tree using 'start tag' events, 'end tag' events,
@@ -69,21 +73,70 @@ class BeautifulSoup(Tag):
     like HTML's <br> tag), call handle_starttag and then
     handle_endtag.
     """
-    ROOT_TAG_NAME = u'[document]'
+    ROOT_TAG_NAME = '[document]'
 
     # If the end-user gives no indication which tree builder they
     # want, look for one with these features.
     DEFAULT_BUILDER_FEATURES = ['html', 'fast']
-
+   
     ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
 
+    NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features=\"%(parser)s\"' to the BeautifulSoup constructor.\n"
+
     def __init__(self, markup="", features=None, builder=None,
-                 parse_only=None, from_encoding=None, **kwargs):
-        """The Soup object is initialized as the 'root tag', and the
-        provided markup (which can be a string or a file-like object)
-        is fed into the underlying parser."""
+                 parse_only=None, from_encoding=None, exclude_encodings=None,
+                 element_classes=None, **kwargs):
+        """Constructor.
+
+        :param markup: A string or a file-like object representing
+        markup to be parsed.
+
+        :param features: Desirable features of the parser to be used. This
+        may be the name of a specific parser ("lxml", "lxml-xml",
+        "html.parser", or "html5lib") or it may be the type of markup
+        to be used ("html", "html5", "xml"). It's recommended that you
+        name a specific parser, so that Beautiful Soup gives you the
+        same results across platforms and virtual environments.
+
+        :param builder: A TreeBuilder subclass to instantiate (or
+        instance to use) instead of looking one up based on
+        `features`. You only need to use this if you've implemented a
+        custom TreeBuilder.
+
+        :param parse_only: A SoupStrainer. Only parts of the document
+        matching the SoupStrainer will be considered. This is useful
+        when parsing part of a document that would otherwise be too
+        large to fit into memory.
+
+        :param from_encoding: A string indicating the encoding of the
+        document to be parsed. Pass this in if Beautiful Soup is
+        guessing wrongly about the document's encoding.
+
+        :param exclude_encodings: A list of strings indicating
+        encodings known to be wrong. Pass this in if you don't know
+        the document's encoding but you know Beautiful Soup's guess is
+        wrong.
+
+        :param element_classes: A dictionary mapping BeautifulSoup
+        classes like Tag and NavigableString to other classes you'd
+        like to be instantiated instead as the parse tree is
+        built. This is useful for using subclasses to modify the
+        default behavior of Tag or NavigableString.
+
+        :param kwargs: For backwards compatibility purposes, the
+        constructor accepts certain keyword arguments used in
+        Beautiful Soup 3. None of these arguments do anything in
+        Beautiful Soup 4; they will result in a warning and then be ignored.
+
+        Apart from this, any keyword arguments passed into the BeautifulSoup
+        constructor are propagated to the TreeBuilder constructor. This
+        makes it possible to configure a TreeBuilder beyond saying
+        which one to use.
+
+        """
 
         if 'convertEntities' in kwargs:
+            del kwargs['convertEntities']
             warnings.warn(
                 "BS4 does not respect the convertEntities argument to the "
                 "BeautifulSoup constructor. Entities are always converted "
@@ -114,9 +167,9 @@ class BeautifulSoup(Tag):
             del kwargs['isHTML']
             warnings.warn(
                 "BS4 does not respect the isHTML argument to the "
-                "BeautifulSoup constructor. You can pass in features='html' "
-                "or features='xml' to get a builder capable of handling "
-                "one or the other.")
+                "BeautifulSoup constructor. Suggest you use "
+                "features='lxml' for HTML and features='lxml-xml' for "
+                "XML.")
 
         def deprecated_argument(old_name, new_name):
             if old_name in kwargs:
@@ -134,13 +187,24 @@ class BeautifulSoup(Tag):
         from_encoding = from_encoding or deprecated_argument(
             "fromEncoding", "from_encoding")
 
-        if len(kwargs) > 0:
-            arg = kwargs.keys().pop()
-            raise TypeError(
-                "__init__() got an unexpected keyword argument '%s'" % arg)
+        if from_encoding and isinstance(markup, str):
+            warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.")
+            from_encoding = None
 
-        if builder is None:
-            if isinstance(features, basestring):
+        self.element_classes = element_classes or dict()
+
+        # We need this information to track whether or not the builder
+        # was specified well enough that we can omit the 'you need to
+        # specify a parser' warning.
+        original_builder = builder
+        original_features = features
+            
+        if isinstance(builder, type):
+            # A builder class was passed in; it needs to be instantiated.
+            builder_class = builder
+            builder = None
+        elif builder is None:
+            if isinstance(features, str):
                 features = [features]
             if features is None or len(features) == 0:
                 features = self.DEFAULT_BUILDER_FEATURES
@@ -150,21 +214,73 @@ class BeautifulSoup(Tag):
                     "Couldn't find a tree builder with the features you "
                     "requested: %s. Do you need to install a parser library?"
                     % ",".join(features))
-            builder = builder_class()
+
+        # At this point either we have a TreeBuilder instance in
+        # builder, or we have a builder_class that we can instantiate
+        # with the remaining **kwargs.
+        if builder is None:
+            builder = builder_class(**kwargs)
+            if not original_builder and not (
+                    original_features == builder.NAME or
+                    original_features in builder.ALTERNATE_NAMES
+            ):
+                if builder.is_xml:
+                    markup_type = "XML"
+                else:
+                    markup_type = "HTML"
+
+                # This code adapted from warnings.py so that we get the same line
+                # of code as our warnings.warn() call gets, even if the answer is wrong
+                # (as it may be in a multithreading situation).
+                caller = None
+                try:
+                    caller = sys._getframe(1)
+                except ValueError:
+                    pass
+                if caller:
+                    globals = caller.f_globals
+                    line_number = caller.f_lineno
+                else:
+                    globals = sys.__dict__
+                    line_number= 1                    
+                filename = globals.get('__file__')
+                if filename:
+                    fnl = filename.lower()
+                    if fnl.endswith((".pyc", ".pyo")):
+                        filename = filename[:-1]
+                if filename:
+                    # If there is no filename at all, the user is most likely in a REPL,
+                    # and the warning is not necessary.
+                    values = dict(
+                        filename=filename,
+                        line_number=line_number,
+                        parser=builder.NAME,
+                        markup_type=markup_type
+                    )
+                    warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % values, stacklevel=2)
+        else:
+            if kwargs:
+                warnings.warn("Keyword arguments to the BeautifulSoup constructor will be ignored. These would normally be passed into the TreeBuilder constructor, but a TreeBuilder instance was passed in as `builder`.")
+                    
         self.builder = builder
         self.is_xml = builder.is_xml
-        self.builder.soup = self
-
+        self.known_xml = self.is_xml
+        self._namespaces = dict()
         self.parse_only = parse_only
 
+        self.builder.initialize_soup(self)
+
         if hasattr(markup, 'read'):        # It's a file-type object.
             markup = markup.read()
-        elif len(markup) <= 256:
+        elif len(markup) <= 256 and (
+                (isinstance(markup, bytes) and not b'<' in markup)
+                or (isinstance(markup, str) and not '<' in markup)
+        ):
             # Print out warnings for a couple beginner problems
             # involving passing non-markup to Beautiful Soup.
             # Beautiful Soup will still parse the input as markup,
             # just in case that's what the user really wants.
-            if (isinstance(markup, unicode)
+            if (isinstance(markup, str)
                 and not os.path.supports_unicode_filenames):
                 possible_filename = markup.encode("utf8")
             else:
@@ -172,37 +288,93 @@ class BeautifulSoup(Tag):
             is_file = False
             try:
                 is_file = os.path.exists(possible_filename)
-            except Exception, e:
+            except Exception as e:
                 # This is almost certainly a problem involving
                 # characters not valid in filenames on this
                 # system. Just let it go.
                 pass
             if is_file:
+                if isinstance(markup, str):
+                    markup = markup.encode("utf8")
                 warnings.warn(
-                    '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup)
-            if markup[:5] == "http:" or markup[:6] == "https:":
-                # TODO: This is ugly but I couldn't get it to work in
-                # Python 3 otherwise.
-                if ((isinstance(markup, bytes) and not b' ' in markup)
-                    or (isinstance(markup, unicode) and not u' ' in markup)):
-                    warnings.warn(
-                        '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)
+                    '"%s" looks like a filename, not markup. You should'
+                    ' probably open this file and pass the filehandle into'
+                    ' Beautiful Soup.' % markup)
+            self._check_markup_is_url(markup)
 
+        rejections = []
+        success = False
         for (self.markup, self.original_encoding, self.declared_html_encoding,
          self.contains_replacement_characters) in (
-            self.builder.prepare_markup(markup, from_encoding)):
+             self.builder.prepare_markup(
+                 markup, from_encoding, exclude_encodings=exclude_encodings)):
             self.reset()
             try:
                 self._feed()
+                success = True
                 break
-            except ParserRejectedMarkup:
+            except ParserRejectedMarkup as e:
+                rejections.append(e)
                 pass
 
+        if not success:
+            other_exceptions = [str(e) for e in rejections]
+            raise ParserRejectedMarkup(
+                "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.\n\nOriginal exception(s) from parser:\n " + "\n ".join(other_exceptions)
+            )
+
         # Clear out the markup and remove the builder's circular
         # reference to this object.
         self.markup = None
         self.builder.soup = None
 
+    def __copy__(self):
+        copy = type(self)(
+            self.encode('utf-8'), builder=self.builder, from_encoding='utf-8'
+        )
+
+        # Although we encoded the tree to UTF-8, that may not have
+        # been the encoding of the original markup. Set the copy's
+        # .original_encoding to reflect the original object's
+        # .original_encoding.
+        copy.original_encoding = self.original_encoding
+        return copy
+
+    def __getstate__(self):
+        # Frequently a tree builder can't be pickled.
+        d = dict(self.__dict__)
+        if 'builder' in d and not self.builder.picklable:
+            d['builder'] = None
+        return d
+
+    @staticmethod
+    def _check_markup_is_url(markup):
+        """ 
+        Check if markup looks like it's actually a url and raise a warning 
+        if so. Markup can be unicode or str (py2) / bytes (py3).
+        """
+        if isinstance(markup, bytes):
+            space = b' '
+            cant_start_with = (b"http:", b"https:")
+        elif isinstance(markup, str):
+            space = ' '
+            cant_start_with = ("http:", "https:")
+        else:
+            return
+
+        if any(markup.startswith(prefix) for prefix in cant_start_with):
+            if not space in markup:
+                if isinstance(markup, bytes):
+                    decoded_markup = markup.decode('utf-8', 'replace')
+                else:
+                    decoded_markup = markup
+                warnings.warn(
+                    '"%s" looks like a URL. Beautiful Soup is not an'
+                    ' HTTP client. You should probably use an HTTP client like'
+                    ' requests to get the document behind the URL, and feed'
+                    ' that document to Beautiful Soup.' % decoded_markup
+                )
+
     def _feed(self):
         # Convert the document to Unicode.
         self.builder.reset()
@@ -223,15 +395,21 @@ class BeautifulSoup(Tag):
         self.preserve_whitespace_tag_stack = []
         self.pushTag(self)
 
-    def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
+    def new_tag(self, name, namespace=None, nsprefix=None, attrs={},
+                sourceline=None, sourcepos=None, **kwattrs):
         """Create a new tag associated with this soup."""
-        return Tag(None, self.builder, name, namespace, nsprefix, attrs)
+        kwattrs.update(attrs)
+        return self.element_classes.get(Tag, Tag)(
+            None, self.builder, name, namespace, nsprefix, kwattrs,
+            sourceline=sourceline, sourcepos=sourcepos
+        )
 
-    def new_string(self, s, subclass=NavigableString):
+    def new_string(self, s, subclass=None):
         """Create a new NavigableString associated with this soup."""
-        navigable = subclass(s)
-        navigable.setup()
-        return navigable
+        subclass = subclass or self.element_classes.get(
+            NavigableString, NavigableString
+        )
+        return subclass(s)
 
     def insert_before(self, successor):
         raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
@@ -250,16 +428,26 @@ class BeautifulSoup(Tag):
 
     def pushTag(self, tag):
         #print "Push", tag.name
-        if self.currentTag:
+        if self.currentTag is not None:
             self.currentTag.contents.append(tag)
         self.tagStack.append(tag)
         self.currentTag = self.tagStack[-1]
         if tag.name in self.builder.preserve_whitespace_tags:
             self.preserve_whitespace_tag_stack.append(tag)
 
-    def endData(self, containerClass=NavigableString):
+    def endData(self, containerClass=None):
+
+        # Default container is NavigableString.
+        containerClass = containerClass or NavigableString
+
+        # The user may want us to instantiate some alias for the
+        # container class.
+        containerClass = self.element_classes.get(
+            containerClass, containerClass
+        )
+        
         if self.current_data:
-            current_data = u''.join(self.current_data)
+            current_data = ''.join(self.current_data)
             # If whitespace is not preserved, and this string contains
             # nothing but ASCII spaces, replace it with a single space
             # or newline.
@@ -289,15 +477,72 @@ class BeautifulSoup(Tag):
 
     def object_was_parsed(self, o, parent=None, most_recent_element=None):
         """Add an object to the parse tree."""
-        parent = parent or self.currentTag
-        most_recent_element = most_recent_element or self._most_recent_element
-        o.setup(parent, most_recent_element)
-
+        if parent is None:
+            parent = self.currentTag
         if most_recent_element is not None:
-            most_recent_element.next_element = o
+            previous_element = most_recent_element
+        else:
+            previous_element = self._most_recent_element
+
+        next_element = previous_sibling = next_sibling = None
+        if isinstance(o, Tag):
+            next_element = o.next_element
+            next_sibling = o.next_sibling
+            previous_sibling = o.previous_sibling
+            if previous_element is None:
+                previous_element = o.previous_element
+
+        fix = parent.next_element is not None
+
+        o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)
+
         self._most_recent_element = o
         parent.contents.append(o)
 
+        # Check if we are inserting into an already parsed node.
+        if fix:
+            self._linkage_fixer(parent)
+
+    def _linkage_fixer(self, el):
+        """Make sure linkage of this fragment is sound."""
+
+        first = el.contents[0]
+        child = el.contents[-1]
+        descendant = child
+
+        if child is first and el.parent is not None:
+            # Parent should be linked to first child
+            el.next_element = child
+            # We are no longer linked to whatever this element is
+            prev_el = child.previous_element
+            if prev_el is not None and prev_el is not el:
+                prev_el.next_element = None
+            # First child should be linked to the parent, and no previous siblings.
+            child.previous_element = el
+            child.previous_sibling = None
+
+        # We have no sibling as we've been appended as the last.
+        child.next_sibling = None
+
+        # This index is a tag, dig deeper for a "last descendant"
+        if isinstance(child, Tag) and child.contents:
+            descendant = child._last_descendant(False)
+
+        # As the final step, link last descendant. It should be linked
+        # to the parent's next sibling (if found), else walk up the chain
+        # and find a parent with a sibling. It should have no next sibling.
+        descendant.next_element = None
+        descendant.next_sibling = None
+        target = el
+        while True:
+            if target is None:
+                break
+            elif target.next_sibling is not None:
+                descendant.next_element = target.next_sibling
+                target.next_sibling.previous_element = child
+                break
+            target = target.parent
+
     def _popToTag(self, name, nsprefix=None, inclusivePop=True):
         """Pops the tag stack up to and including the most recent
         instance of the given tag. If inclusivePop is false, pops the tag
@@ -321,11 +566,12 @@ class BeautifulSoup(Tag):
 
         return most_recently_popped
 
-    def handle_starttag(self, name, namespace, nsprefix, attrs):
+    def handle_starttag(self, name, namespace, nsprefix, attrs, sourceline=None,
+                        sourcepos=None):
         """Push a start tag on to the stack.
 
         If this method returns None, the tag was rejected by the
-        SoupStrainer. You should proceed as if the tag had not occured
+        SoupStrainer. You should proceed as if the tag had not occurred
         in the document. For instance, if this was a self-closing tag,
         don't call handle_endtag.
         """
@@ -338,11 +584,14 @@ class BeautifulSoup(Tag):
                  or not self.parse_only.search_tag(name, attrs))):
             return None
 
-        tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
-                  self.currentTag, self._most_recent_element)
+        tag = self.element_classes.get(Tag, Tag)(
+            self, self.builder, name, namespace, nsprefix, attrs,
+            self.currentTag, self._most_recent_element,
+            sourceline=sourceline, sourcepos=sourcepos
+        )
         if tag is None:
             return tag
-        if self._most_recent_element:
+        if self._most_recent_element is not None:
             self._most_recent_element.next_element = tag
         self._most_recent_element = tag
         self.pushTag(tag)
@@ -367,9 +616,9 @@ class BeautifulSoup(Tag):
             encoding_part = ''
             if eventual_encoding != None:
                 encoding_part = ' encoding="%s"' % eventual_encoding
-            prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
+            prefix = '<?xml version="1.0"%s?>\n' % encoding_part
         else:
-            prefix = u''
+            prefix = ''
         if not pretty_print:
             indent_level = None
         else:
@@ -403,4 +652,4 @@ class FeatureNotFound(ValueError):
 if __name__ == '__main__':
     import sys
     soup = BeautifulSoup(sys.stdin)
-    print soup.prettify()
+    print(soup.prettify())

lib/bs4/builder/__init__.py

@@ -1,10 +1,13 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 from collections import defaultdict
 import itertools
 import sys
 from bs4.element import (
     CharsetMetaAttributeValue,
     ContentMetaAttributeValue,
-    whitespace_re
+    nonwhitespace_re
     )
 
 __all__ = [
@@ -80,21 +83,70 @@ builder_registry = TreeBuilderRegistry()
 class TreeBuilder(object):
     """Turn a document into a Beautiful Soup object tree."""
 
+    NAME = "[Unknown tree builder]"
+    ALTERNATE_NAMES = []
     features = []
 
     is_xml = False
-    preserve_whitespace_tags = set()
+    picklable = False
     empty_element_tags = None # A tag will be considered an empty-element
                               # tag when and only when it has no contents.
-
+    
     # A value for these tag/attribute combinations is a space- or
     # comma-separated list of CDATA, rather than a single CDATA.
-    cdata_list_attributes = {}
+    DEFAULT_CDATA_LIST_ATTRIBUTES = {}
 
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set()
+    
+    USE_DEFAULT = object()
 
-    def __init__(self):
+    # Most parsers don't keep track of line numbers.
+    TRACKS_LINE_NUMBERS = False
+    
+    def __init__(self, multi_valued_attributes=USE_DEFAULT,
+                 preserve_whitespace_tags=USE_DEFAULT,
+                 store_line_numbers=USE_DEFAULT):
+        """Constructor.
+
+        :param multi_valued_attributes: If this is set to None, the
+        TreeBuilder will not turn any values for attributes like
+        'class' into lists. Setting this do a dictionary will
+        customize this behavior; look at DEFAULT_CDATA_LIST_ATTRIBUTES
+        for an example.
+
+        Internally, these are called "CDATA list attributes", but that
+        probably doesn't make sense to an end-user, so the argument name
+        is `multi_valued_attributes`.
+
+        :param preserve_whitespace_tags: A list of tags to treat
+        the way <pre> tags are treated in HTML. Tags in this list
+        will have 
+
+        :param store_line_numbers: If the parser keeps track of the
+        line numbers and positions of the original markup, that
+        information will, by default, be stored in each corresponding
+        `Tag` object. You can turn this off by passing
+        store_line_numbers=False. If the parser you're using doesn't 
+        keep track of this information, then setting store_line_numbers=True
+        will do nothing.
+        """
         self.soup = None
-
+        if multi_valued_attributes is self.USE_DEFAULT:
+            multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES
+        self.cdata_list_attributes = multi_valued_attributes
+        if preserve_whitespace_tags is self.USE_DEFAULT:
+            preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS
+        self.preserve_whitespace_tags = preserve_whitespace_tags
+        if store_line_numbers == self.USE_DEFAULT:
+            store_line_numbers = self.TRACKS_LINE_NUMBERS
+        self.store_line_numbers = store_line_numbers
+        
+    def initialize_soup(self, soup):
+        """The BeautifulSoup object has been initialized and is now
+        being associated with the TreeBuilder.
+        """
+        self.soup = soup
+        
     def reset(self):
         pass
 
@@ -118,13 +170,13 @@ class TreeBuilder(object):
         if self.empty_element_tags is None:
             return True
         return tag_name in self.empty_element_tags
-
+    
     def feed(self, markup):
         raise NotImplementedError()
 
     def prepare_markup(self, markup, user_specified_encoding=None,
-                       document_declared_encoding=None):
-        return markup, None, None, False
+                       document_declared_encoding=None, exclude_encodings=None):
+        yield markup, None, None, False
 
     def test_fragment_to_document(self, fragment):
         """Wrap an HTML fragment to make it look like a document.
@@ -153,14 +205,14 @@ class TreeBuilder(object):
             universal = self.cdata_list_attributes.get('*', [])
             tag_specific = self.cdata_list_attributes.get(
                 tag_name.lower(), None)
-            for attr in attrs.keys():
+            for attr in list(attrs.keys()):
                 if attr in universal or (tag_specific and attr in tag_specific):
                     # We have a "class"-type attribute whose string
                     # value is a whitespace-separated list of
                     # values. Split it into a list.
                     value = attrs[attr]
-                    if isinstance(value, basestring):
-                        values = whitespace_re.split(value)
+                    if isinstance(value, str):
+                        values = nonwhitespace_re.findall(value)
                     else:
                         # html5lib sometimes calls setAttributes twice
                         # for the same tag when rearranging the parse
@@ -224,10 +276,20 @@ class HTMLTreeBuilder(TreeBuilder):
     Such as which tags are empty-element tags.
     """
 
-    preserve_whitespace_tags = set(['pre', 'textarea'])
-    empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
-                              'spacer', 'link', 'frame', 'base'])
+    empty_element_tags = set([
+        # These are from HTML5.
+        'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
+        
+        # These are from earlier versions of HTML and are removed in HTML5.
+        'basefont', 'bgsound', 'command', 'frame', 'image', 'isindex', 'nextid', 'spacer'
+    ])
 
+    # The HTML standard defines these as block-level elements. Beautiful
+    # Soup does not treat these elements differently from other elements,
+    # but it may do so eventually, and this information is available if
+    # you need to use it.
+    block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"])
+    
     # The HTML standard defines these attributes as containing a
     # space-separated list of values, not a single value. That is,
     # class="foo bar" means that the 'class' attribute has two values,
@@ -235,7 +297,7 @@ class HTMLTreeBuilder(TreeBuilder):
     # encounter one of these attributes, we will parse its value into
     # a list of values if possible. Upon output, the list will be
     # converted back into a string.
-    cdata_list_attributes = {
+    DEFAULT_CDATA_LIST_ATTRIBUTES = {
         "*" : ['class', 'accesskey', 'dropzone'],
         "a" : ['rel', 'rev'],
         "link" :  ['rel', 'rev'],
@@ -252,6 +314,8 @@ class HTMLTreeBuilder(TreeBuilder):
         "output" : ["for"],
         }
 
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
+    
     def set_up_substitutions(self, tag):
         # We are only interested in <meta> tags
         if tag.name != 'meta':
@@ -299,8 +363,15 @@ def register_treebuilders_from(module):
             this_module.builder_registry.register(obj)
 
 class ParserRejectedMarkup(Exception):
-    pass
-
+    def __init__(self, message_or_exception):
+        """Explain why the parser rejected the given markup, either
+        with a textual explanation or another exception.
+        """
+        if isinstance(message_or_exception, Exception):
+            e = message_or_exception
+            message_or_exception = "%s: %s" % (e.__class__.__name__, str(e))
+        super(ParserRejectedMarkup, self).__init__(message_or_exception)
+            
 # Builders are registered in reverse order of priority, so that custom
 # builder registrations will take precedence. In general, we want lxml
 # to take precedence over html5lib, because it's faster. And we only

lib/bs4/builder/_html5lib.py

@@ -1,17 +1,27 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'HTML5TreeBuilder',
     ]
 
 import warnings
+import re
 from bs4.builder import (
     PERMISSIVE,
     HTML,
     HTML_5,
     HTMLTreeBuilder,
     )
-from bs4.element import NamespacedAttribute
+from bs4.element import (
+    NamespacedAttribute,
+    nonwhitespace_re,
+)
 import html5lib
-from html5lib.constants import namespaces
+from html5lib.constants import (
+    namespaces,
+    prefixes,
+    )
 from bs4.element import (
     Comment,
     Doctype,
@@ -19,14 +29,36 @@ from bs4.element import (
     Tag,
     )
 
+try:
+    # Pre-0.99999999
+    from html5lib.treebuilders import _base as treebuilder_base
+    new_html5lib = False
+except ImportError as e:
+    # 0.99999999 and up
+    from html5lib.treebuilders import base as treebuilder_base
+    new_html5lib = True
+
 class HTML5TreeBuilder(HTMLTreeBuilder):
     """Use html5lib to build a tree."""
 
-    features = ['html5lib', PERMISSIVE, HTML_5, HTML]
+    NAME = "html5lib"
 
-    def prepare_markup(self, markup, user_specified_encoding):
+    features = [NAME, PERMISSIVE, HTML_5, HTML]
+
+    # html5lib can tell us which line number and position in the
+    # original file is the source of an element.
+    TRACKS_LINE_NUMBERS = True
+    
+    def prepare_markup(self, markup, user_specified_encoding,
+                       document_declared_encoding=None, exclude_encodings=None):
         # Store the user-specified encoding for use later on.
         self.user_specified_encoding = user_specified_encoding
+
+        # document_declared_encoding and exclude_encodings aren't used
+        # ATM because the html5lib TreeBuilder doesn't use
+        # UnicodeDammit.
+        if exclude_encodings:
+            warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
         yield (markup, None, None, False)
 
     # These methods are defined by Beautiful Soup.
@@ -34,32 +66,63 @@ class HTML5TreeBuilder(HTMLTreeBuilder):
         if self.soup.parse_only is not None:
             warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
         parser = html5lib.HTMLParser(tree=self.create_treebuilder)
-        doc = parser.parse(markup, encoding=self.user_specified_encoding)
-
+        self.underlying_builder.parser = parser
+        extra_kwargs = dict()
+        if not isinstance(markup, str):
+            if new_html5lib:
+                extra_kwargs['override_encoding'] = self.user_specified_encoding
+            else:
+                extra_kwargs['encoding'] = self.user_specified_encoding
+        doc = parser.parse(markup, **extra_kwargs)
+        
         # Set the character encoding detected by the tokenizer.
-        if isinstance(markup, unicode):
+        if isinstance(markup, str):
             # We need to special-case this because html5lib sets
             # charEncoding to UTF-8 if it gets Unicode input.
             doc.original_encoding = None
         else:
-            doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
-
+            original_encoding = parser.tokenizer.stream.charEncoding[0]
+            if not isinstance(original_encoding, str):
+                # In 0.99999999 and up, the encoding is an html5lib
+                # Encoding object. We want to use a string for compatibility
+                # with other tree builders.
+                original_encoding = original_encoding.name
+            doc.original_encoding = original_encoding
+        self.underlying_builder.parser = None
+            
     def create_treebuilder(self, namespaceHTMLElements):
         self.underlying_builder = TreeBuilderForHtml5lib(
-            self.soup, namespaceHTMLElements)
+            namespaceHTMLElements, self.soup,
+            store_line_numbers=self.store_line_numbers
+        )
         return self.underlying_builder
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""
-        return u'<html><head></head><body>%s</body></html>' % fragment
+        return '<html><head></head><body>%s</body></html>' % fragment
 
 
-class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
-
-    def __init__(self, soup, namespaceHTMLElements):
-        self.soup = soup
+class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
+    
+    def __init__(self, namespaceHTMLElements, soup=None,
+                 store_line_numbers=True, **kwargs):
+        if soup:
+            self.soup = soup
+        else:
+            from bs4 import BeautifulSoup
+            # TODO: Why is the parser 'html.parser' here? To avoid an
+            # infinite loop?
+            self.soup = BeautifulSoup(
+                "", "html.parser", store_line_numbers=store_line_numbers,
+                **kwargs
+            )
         super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
 
+        # This will be set later to an html5lib.html5parser.HTMLParser
+        # object, which we can use to track the current line number.
+        self.parser = None
+        self.store_line_numbers = store_line_numbers
+        
     def documentClass(self):
         self.soup.reset()
         return Element(self.soup, self.soup, None)
@@ -73,14 +136,26 @@ class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
         self.soup.object_was_parsed(doctype)
 
     def elementClass(self, name, namespace):
-        tag = self.soup.new_tag(name, namespace)
+        kwargs = {}
+        if self.parser and self.store_line_numbers:
+            # This represents the point immediately after the end of the
+            # tag. We don't know when the tag started, but we do know
+            # where it ended -- the character just before this one.
+            sourceline, sourcepos = self.parser.tokenizer.stream.position()
+            kwargs['sourceline'] = sourceline
+            kwargs['sourcepos'] = sourcepos-1
+        tag = self.soup.new_tag(name, namespace, **kwargs)
+
         return Element(tag, self.soup, namespace)
 
     def commentClass(self, data):
         return TextNode(Comment(data), self.soup)
 
     def fragmentClass(self):
-        self.soup = BeautifulSoup("")
+        from bs4 import BeautifulSoup
+        # TODO: Why is the parser 'html.parser' here? To avoid an
+        # infinite loop?
+        self.soup = BeautifulSoup("", "html.parser")
         self.soup.name = "[document_fragment]"
         return Element(self.soup, self.soup, None)
 
@@ -92,7 +167,57 @@ class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
         return self.soup
 
     def getFragment(self):
-        return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
+        return treebuilder_base.TreeBuilder.getFragment(self).element
+
+    def testSerializer(self, element):
+        from bs4 import BeautifulSoup
+        rv = []
+        doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$')
+
+        def serializeElement(element, indent=0):
+            if isinstance(element, BeautifulSoup):
+                pass
+            if isinstance(element, Doctype):
+                m = doctype_re.match(element)
+                if m:
+                    name = m.group(1)
+                    if m.lastindex > 1:
+                        publicId = m.group(2) or ""
+                        systemId = m.group(3) or m.group(4) or ""
+                        rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (' ' * indent, name, publicId, systemId))
+                    else:
+                        rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name))
+                else:
+                    rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
+            elif isinstance(element, Comment):
+                rv.append("|%s<!-- %s -->" % (' ' * indent, element))
+            elif isinstance(element, NavigableString):
+                rv.append("|%s\"%s\"" % (' ' * indent, element))
+            else:
+                if element.namespace:
+                    name = "%s %s" % (prefixes[element.namespace],
+                                      element.name)
+                else:
+                    name = element.name
+                rv.append("|%s<%s>" % (' ' * indent, name))
+                if element.attrs:
+                    attributes = []
+                    for name, value in list(element.attrs.items()):
+                        if isinstance(name, NamespacedAttribute):
+                            name = "%s %s" % (prefixes[name.namespace], name.name)
+                        if isinstance(value, list):
+                            value = " ".join(value)
+                        attributes.append((name, value))
+
+                    for name, value in sorted(attributes):
+                        rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
+                indent += 2
+                for child in element.children:
+                    serializeElement(child, indent)
+        serializeElement(element, 0)
+
+        return "\n".join(rv)
 
 class AttrList(object):
     def __init__(self, element):
@@ -101,7 +226,16 @@ class AttrList(object):
     def __iter__(self):
         return list(self.attrs.items()).__iter__()
     def __setitem__(self, name, value):
-        "set attr", name, value
+        # If this attribute is a multi-valued attribute for this element,
+        # turn its value into a list.
+        list_attr = self.element.cdata_list_attributes
+        if (name in list_attr['*']
+            or (self.element.name in list_attr
+                and name in list_attr[self.element.name])):
+            # A node that is being cloned may have already undergone
+            # this procedure.
+            if not isinstance(value, list):
+                value = nonwhitespace_re.findall(value)
         self.element[name] = value
     def items(self):
         return list(self.attrs.items())
@@ -115,16 +249,16 @@ class AttrList(object):
         return name in list(self.attrs.keys())
 
 
-class Element(html5lib.treebuilders._base.Node):
+class Element(treebuilder_base.Node):
     def __init__(self, element, soup, namespace):
-        html5lib.treebuilders._base.Node.__init__(self, element.name)
+        treebuilder_base.Node.__init__(self, element.name)
         self.element = element
         self.soup = soup
         self.namespace = namespace
 
     def appendChild(self, node):
         string_child = child = None
-        if isinstance(node, basestring):
+        if isinstance(node, str):
             # Some other piece of code decided to pass in a string
             # instead of creating a TextElement object to contain the
             # string.
@@ -136,13 +270,15 @@ class Element(html5lib.treebuilders._base.Node):
             child = node
         elif node.element.__class__ == NavigableString:
             string_child = child = node.element
+            node.parent = self
         else:
             child = node.element
+            node.parent = self
 
-        if not isinstance(child, basestring) and child.parent is not None:
+        if not isinstance(child, str) and child.parent is not None:
             node.element.extract()
 
-        if (string_child and self.element.contents
+        if (string_child is not None and self.element.contents
             and self.element.contents[-1].__class__ == NavigableString):
             # We are appending a string onto another string.
             # TODO This has O(n^2) performance, for input like
@@ -152,7 +288,7 @@ class Element(html5lib.treebuilders._base.Node):
             old_element.replace_with(new_element)
             self.soup._most_recent_element = new_element
         else:
-            if isinstance(node, basestring):
+            if isinstance(node, str):
                 # Create a brand new NavigableString from this string.
                 child = self.soup.new_string(node)
 
@@ -161,6 +297,12 @@ class Element(html5lib.treebuilders._base.Node):
             # immediately after the parent, if it has no children.)
             if self.element.contents:
                 most_recent_element = self.element._last_descendant(False)
+            elif self.element.next_element is not None:
+                # Something from further ahead in the parse tree is
+                # being inserted into this earlier element. This is
+                # very annoying because it means an expensive search
+                # for the last element in the tree.
+                most_recent_element = self.soup._last_descendant()
             else:
                 most_recent_element = self.element
 
@@ -169,9 +311,12 @@ class Element(html5lib.treebuilders._base.Node):
                 most_recent_element=most_recent_element)
 
     def getAttributes(self):
+        if isinstance(self.element, Comment):
+            return {}
         return AttrList(self.element)
 
     def setAttributes(self, attributes):
+
         if attributes is not None and len(attributes) > 0:
 
             converted_attributes = []
@@ -183,7 +328,7 @@ class Element(html5lib.treebuilders._base.Node):
 
             self.soup.builder._replace_cdata_list_attribute_values(
                 self.name, attributes)
-            for name, value in attributes.items():
+            for name, value in list(attributes.items()):
                 self.element[name] = value
 
             # The attributes may contain variables that need substitution.
@@ -195,11 +340,11 @@ class Element(html5lib.treebuilders._base.Node):
     attributes = property(getAttributes, setAttributes)
 
     def insertText(self, data, insertBefore=None):
+        text = TextNode(self.soup.new_string(data), self.soup)
         if insertBefore:
-            text = TextNode(self.soup.new_string(data), self.soup)
-            self.insertBefore(data, insertBefore)
+            self.insertBefore(text, insertBefore)
         else:
-            self.appendChild(data)
+            self.appendChild(text)
 
     def insertBefore(self, node, refNode):
         index = self.element.index(refNode.element)
@@ -218,6 +363,10 @@ class Element(html5lib.treebuilders._base.Node):
 
     def reparentChildren(self, new_parent):
         """Move all of this tag's children into another tag."""
+        # print "MOVE", self.element.contents
+        # print "FROM", self.element
+        # print "TO", new_parent.element
+
         element = self.element
         new_parent_element = new_parent.element
         # Determine what this tag's next_element will be once all the children
@@ -236,18 +385,35 @@ class Element(html5lib.treebuilders._base.Node):
             new_parents_last_descendant_next_element = new_parent_element.next_element
 
         to_append = element.contents
-        append_after = new_parent.element.contents
         if len(to_append) > 0:
             # Set the first child's previous_element and previous_sibling
             # to elements within the new parent
             first_child = to_append[0]
-            first_child.previous_element = new_parents_last_descendant
+            if new_parents_last_descendant is not None:
+                first_child.previous_element = new_parents_last_descendant
+            else:
+                first_child.previous_element = new_parent_element
             first_child.previous_sibling = new_parents_last_child
+            if new_parents_last_descendant is not None:
+                new_parents_last_descendant.next_element = first_child
+            else:
+                new_parent_element.next_element = first_child
+            if new_parents_last_child is not None:
+                new_parents_last_child.next_sibling = first_child
 
-            # Fix the last child's next_element and next_sibling
-            last_child = to_append[-1]
-            last_child.next_element = new_parents_last_descendant_next_element
-            last_child.next_sibling = None
+            # Find the very last element being moved. It is now the
+            # parent's last descendant. It has no .next_sibling and
+            # its .next_element is whatever the previous last
+            # descendant had.
+            last_childs_last_descendant = to_append[-1]._last_descendant(False, True)
+
+            last_childs_last_descendant.next_element = new_parents_last_descendant_next_element
+            if new_parents_last_descendant_next_element is not None:
+                # TODO: This code has no test coverage and I'm not sure
+                # how to get html5lib to go through this path, but it's
+                # just the other side of the previous line.
+                new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant
+            last_childs_last_descendant.next_sibling = None
 
         for child in to_append:
             child.parent = new_parent_element
@@ -257,6 +423,10 @@ class Element(html5lib.treebuilders._base.Node):
         element.contents = []
         element.next_element = final_next_element
 
+        # print "DONE WITH MOVE"
+        # print "FROM", self.element
+        # print "TO", new_parent_element
+
     def cloneNode(self):
         tag = self.soup.new_tag(self.element.name, self.namespace)
         node = Element(tag, self.soup, self.namespace)
@@ -268,7 +438,7 @@ class Element(html5lib.treebuilders._base.Node):
         return self.element.contents
 
     def getNameTuple(self):
-        if self.namespace is None:
+        if self.namespace == None:
             return namespaces["html"], self.name
         else:
             return self.namespace, self.name
@@ -277,7 +447,7 @@ class Element(html5lib.treebuilders._base.Node):
 
 class TextNode(Element):
     def __init__(self, element, soup):
-        html5lib.treebuilders._base.Node.__init__(self, None)
+        treebuilder_base.Node.__init__(self, None)
         self.element = element
         self.soup = soup
 

lib/bs4/builder/_htmlparser.py

@@ -1,13 +1,23 @@
+# encoding: utf-8
 """Use the HTMLParser library to parse HTML files that aren't too bad."""
 
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'HTMLParserTreeBuilder',
     ]
 
-from HTMLParser import (
-    HTMLParser,
-    HTMLParseError,
-    )
+from future.moves.html.parser import HTMLParser
+
+try:
+    from html.parser import HTMLParseError
+except ImportError as e:
+    # HTMLParseError is removed in Python 3.5. Since it can never be
+    # thrown in 3.5, we can just define our own class as a placeholder.
+    class HTMLParseError(Exception):
+        pass
+
 import sys
 import warnings
 
@@ -19,10 +29,10 @@ import warnings
 # At the end of this file, we monkeypatch HTMLParser so that
 # strict=True works well on Python 3.2.2.
 major, minor, release = sys.version_info[:3]
-CONSTRUCTOR_TAKES_STRICT = (
-    major > 3
-    or (major == 3 and minor > 2)
-    or (major == 3 and minor == 2 and release >= 3))
+CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3
+CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3
+CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4
+
 
 from bs4.element import (
     CData,
@@ -43,7 +53,42 @@ from bs4.builder import (
 HTMLPARSER = 'html.parser'
 
 class BeautifulSoupHTMLParser(HTMLParser):
-    def handle_starttag(self, name, attrs):
+
+    def __init__(self, *args, **kwargs):
+        HTMLParser.__init__(self, *args, **kwargs)
+
+        # Keep a list of empty-element tags that were encountered
+        # without an explicit closing tag. If we encounter a closing tag
+        # of this type, we'll associate it with one of those entries.
+        #
+        # This isn't a stack because we don't care about the
+        # order. It's a list of closing tags we've already handled and
+        # will ignore, assuming they ever show up.
+        self.already_closed_empty_element = []
+
+    def error(self, msg):
+        """In Python 3, HTMLParser subclasses must implement error(), although this
+        requirement doesn't appear to be documented.
+
+        In Python 2, HTMLParser implements error() as raising an exception.
+
+        In any event, this method is called only on very strange markup and our best strategy
+        is to pretend it didn't happen and keep going.
+        """
+        warnings.warn(msg)
+        
+    def handle_startendtag(self, name, attrs):
+        # This is only called when the markup looks like
+        # <tag/>.
+
+        # is_startend() tells handle_starttag not to close the tag
+        # just because its name matches a known empty-element tag. We
+        # know that this is an empty-element tag and we want to call
+        # handle_endtag ourselves.
+        tag = self.handle_starttag(name, attrs, handle_empty_element=False)
+        self.handle_endtag(name)
+        
+    def handle_starttag(self, name, attrs, handle_empty_element=True):
         # XXX namespace
         attr_dict = {}
         for key, value in attrs:
@@ -53,17 +98,46 @@ class BeautifulSoupHTMLParser(HTMLParser):
                 value = ''
             attr_dict[key] = value
             attrvalue = '""'
-        self.soup.handle_starttag(name, None, None, attr_dict)
+        #print "START", name
+        sourceline, sourcepos = self.getpos()
+        tag = self.soup.handle_starttag(
+            name, None, None, attr_dict, sourceline=sourceline,
+            sourcepos=sourcepos
+        )
+        if tag and tag.is_empty_element and handle_empty_element:
+            # Unlike other parsers, html.parser doesn't send separate end tag
+            # events for empty-element tags. (It's handled in
+            # handle_startendtag, but only if the original markup looked like
+            # <tag/>.)
+            #
+            # So we need to call handle_endtag() ourselves. Since we
+            # know the start event is identical to the end event, we
+            # don't want handle_endtag() to cross off any previous end
+            # events for tags of this name.
+            self.handle_endtag(name, check_already_closed=False)
 
-    def handle_endtag(self, name):
-        self.soup.handle_endtag(name)
+            # But we might encounter an explicit closing tag for this tag
+            # later on. If so, we want to ignore it.
+            self.already_closed_empty_element.append(name)
+            
+    def handle_endtag(self, name, check_already_closed=True):
+        #print "END", name
+        if check_already_closed and name in self.already_closed_empty_element:
+            # This is a redundant end tag for an empty-element tag.
+            # We've already called handle_endtag() for it, so just
+            # check it off the list.
+            # print "ALREADY CLOSED", name
+            self.already_closed_empty_element.remove(name)
+        else:
+            self.soup.handle_endtag(name)
 
     def handle_data(self, data):
         self.soup.handle_data(data)
 
     def handle_charref(self, name):
         # XXX workaround for a bug in HTMLParser. Remove this once
-        # it's fixed.
+        # it's fixed in all supported versions.
+        # http://bugs.python.org/issue13633
         if name.startswith('x'):
             real_name = int(name.lstrip('x'), 16)
         elif name.startswith('X'):
@@ -71,11 +145,26 @@ class BeautifulSoupHTMLParser(HTMLParser):
         else:
             real_name = int(name)
 
-        try:
-            data = unichr(real_name)
-        except (ValueError, OverflowError), e:
-            data = u"\N{REPLACEMENT CHARACTER}"
-
+        data = None
+        if real_name < 256:
+            # HTML numeric entities are supposed to reference Unicode
+            # code points, but sometimes they reference code points in
+            # some other encoding (ahem, Windows-1252). E.g. &#147;
+            # instead of &#201; for LEFT DOUBLE QUOTATION MARK. This
+            # code tries to detect this situation and compensate.
+            for encoding in (self.soup.original_encoding, 'windows-1252'):
+                if not encoding:
+                    continue
+                try:
+                    data = bytearray([real_name]).decode(encoding)
+                except UnicodeDecodeError as e:
+                    pass
+        if not data:
+            try:
+                data = chr(real_name)
+            except (ValueError, OverflowError) as e:
+                pass
+        data = data or "\N{REPLACEMENT CHARACTER}"
         self.handle_data(data)
 
     def handle_entityref(self, name):
@@ -83,7 +172,12 @@ class BeautifulSoupHTMLParser(HTMLParser):
         if character is not None:
             data = character
         else:
-            data = "&%s;" % name
+            # If this were XML, it would be ambiguous whether "&foo"
+            # was an character entity reference with a missing
+            # semicolon or the literal string "&foo". Since this is
+            # HTML, we have a complete list of all character entity references,
+            # and this one wasn't found, so assume it's the literal string "&foo".
+            data = "&%s" % name
         self.handle_data(data)
 
     def handle_comment(self, data):
@@ -113,14 +207,6 @@ class BeautifulSoupHTMLParser(HTMLParser):
 
     def handle_pi(self, data):
         self.soup.endData()
-        if data.endswith("?") and data.lower().startswith("xml"):
-            # "An XHTML processing instruction using the trailing '?'
-            # will cause the '?' to be included in data." - HTMLParser
-            # docs.
-            #
-            # Strip the question mark so we don't end up with two
-            # question marks.
-            data = data[:-1]
         self.soup.handle_data(data)
         self.soup.endData(ProcessingInstruction)
 
@@ -128,26 +214,38 @@ class BeautifulSoupHTMLParser(HTMLParser):
 class HTMLParserTreeBuilder(HTMLTreeBuilder):
 
     is_xml = False
-    features = [HTML, STRICT, HTMLPARSER]
+    picklable = True
+    NAME = HTMLPARSER
+    features = [NAME, HTML, STRICT]
 
-    def __init__(self, *args, **kwargs):
-        if CONSTRUCTOR_TAKES_STRICT:
-            kwargs['strict'] = False
-        self.parser_args = (args, kwargs)
+    # The html.parser knows which line number and position in the
+    # original file is the source of an element.
+    TRACKS_LINE_NUMBERS = True
+    
+    def __init__(self, parser_args=None, parser_kwargs=None, **kwargs):
+        super(HTMLParserTreeBuilder, self).__init__(**kwargs)
+        parser_args = parser_args or []
+        parser_kwargs = parser_kwargs or {}
+        if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED:
+            parser_kwargs['strict'] = False
+        if CONSTRUCTOR_TAKES_CONVERT_CHARREFS:
+            parser_kwargs['convert_charrefs'] = False
+        self.parser_args = (parser_args, parser_kwargs)
 
     def prepare_markup(self, markup, user_specified_encoding=None,
-                       document_declared_encoding=None):
+                       document_declared_encoding=None, exclude_encodings=None):
         """
         :return: A 4-tuple (markup, original encoding, encoding
         declared within markup, whether any characters had to be
         replaced with REPLACEMENT CHARACTER).
         """
-        if isinstance(markup, unicode):
+        if isinstance(markup, str):
             yield (markup, None, None, False)
             return
 
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
+        dammit = UnicodeDammit(markup, try_encodings, is_html=True,
+                               exclude_encodings=exclude_encodings)
         yield (dammit.markup, dammit.original_encoding,
                dammit.declared_html_encoding,
                dammit.contains_replacement_characters)
@@ -158,10 +256,12 @@ class HTMLParserTreeBuilder(HTMLTreeBuilder):
         parser.soup = self.soup
         try:
             parser.feed(markup)
-        except HTMLParseError, e:
+            parser.close()
+        except HTMLParseError as e:
             warnings.warn(RuntimeWarning(
                 "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
             raise e
+        parser.already_closed_empty_element = []
 
 # Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
 # 3.2.3 code. This ensures they don't treat markup like <p></p> as a

lib/bs4/builder/_lxml.py

@@ -1,13 +1,26 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'LXMLTreeBuilderForXML',
     'LXMLTreeBuilder',
     ]
 
+try:
+    from collections.abc import Callable # Python 3.6
+except ImportError as e:
+    from collections import Callable
+
 from io import BytesIO
-from StringIO import StringIO
-import collections
+from io import StringIO
 from lxml import etree
-from bs4.element import Comment, Doctype, NamespacedAttribute
+from bs4.element import (
+    Comment,
+    Doctype,
+    NamespacedAttribute,
+    ProcessingInstruction,
+    XMLProcessingInstruction,
+)
 from bs4.builder import (
     FAST,
     HTML,
@@ -20,19 +33,55 @@ from bs4.dammit import EncodingDetector
 
 LXML = 'lxml'
 
+def _invert(d):
+    "Invert a dictionary."
+    return dict((v,k) for k, v in list(d.items()))
+
 class LXMLTreeBuilderForXML(TreeBuilder):
     DEFAULT_PARSER_CLASS = etree.XMLParser
 
     is_xml = True
+    processing_instruction_class = XMLProcessingInstruction
+
+    NAME = "lxml-xml"
+    ALTERNATE_NAMES = ["xml"]
 
     # Well, it's permissive by XML parser standards.
-    features = [LXML, XML, FAST, PERMISSIVE]
+    features = [NAME, LXML, XML, FAST, PERMISSIVE]
 
     CHUNK_SIZE = 512
 
     # This namespace mapping is specified in the XML Namespace
     # standard.
-    DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"}
+    DEFAULT_NSMAPS = dict(xml='http://www.w3.org/XML/1998/namespace')
+
+    DEFAULT_NSMAPS_INVERTED = _invert(DEFAULT_NSMAPS)
+
+    # NOTE: If we parsed Element objects and looked at .sourceline,
+    # we'd be able to see the line numbers from the original document.
+    # But instead we build an XMLParser or HTMLParser object to serve
+    # as the target of parse messages, and those messages don't include
+    # line numbers.
+    
+    def initialize_soup(self, soup):
+        """Let the BeautifulSoup object know about the standard namespace
+        mapping.
+        """
+        super(LXMLTreeBuilderForXML, self).initialize_soup(soup)
+        self._register_namespaces(self.DEFAULT_NSMAPS)
+
+    def _register_namespaces(self, mapping):
+        """Let the BeautifulSoup object know about namespaces encountered
+        while parsing the document.
+
+        This might be useful later on when creating CSS selectors.
+        """
+        for key, value in list(mapping.items()):
+            if key and key not in self.soup._namespaces:
+                # Let the BeautifulSoup object know about a new namespace.
+                # If there are multiple namespaces defined with the same
+                # prefix, the first one in the document takes precedence.
+                self.soup._namespaces[key] = value
 
     def default_parser(self, encoding):
         # This can either return a parser object or a class, which
@@ -46,12 +95,12 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         # Use the default parser.
         parser = self.default_parser(encoding)
 
-        if isinstance(parser, collections.Callable):
+        if isinstance(parser, Callable):
             # Instantiate the parser with default arguments
             parser = parser(target=self, strip_cdata=False, encoding=encoding)
         return parser
 
-    def __init__(self, parser=None, empty_element_tags=None):
+    def __init__(self, parser=None, empty_element_tags=None, **kwargs):
         # TODO: Issue a warning if parser is present but not a
         # callable, since that means there's no way to create new
         # parsers for different encodings.
@@ -59,8 +108,9 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         if empty_element_tags is not None:
             self.empty_element_tags = set(empty_element_tags)
         self.soup = None
-        self.nsmaps = [self.DEFAULT_NSMAPS]
-
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
+        super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
+        
     def _getNsTag(self, tag):
         # Split the namespace URL out of a fully-qualified lxml tag
         # name. Copied from lxml's src/lxml/sax.py.
@@ -70,6 +120,7 @@ class LXMLTreeBuilderForXML(TreeBuilder):
             return (None, tag)
 
     def prepare_markup(self, markup, user_specified_encoding=None,
+                       exclude_encodings=None,
                        document_declared_encoding=None):
         """
         :yield: A series of 4-tuples.
@@ -78,31 +129,37 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
         Each 4-tuple represents a strategy for parsing the document.
         """
-        if isinstance(markup, unicode):
-            # We were given Unicode. Maybe lxml can parse Unicode on
-            # this system?
-            yield markup, None, document_declared_encoding, False
-
-        if isinstance(markup, unicode):
-            # No, apparently not. Convert the Unicode to UTF-8 and
-            # tell lxml to parse it as UTF-8.
-            yield (markup.encode("utf8"), "utf8",
-                   document_declared_encoding, False)
-
         # Instead of using UnicodeDammit to convert the bytestring to
         # Unicode using different encodings, use EncodingDetector to
         # iterate over the encodings, and tell lxml to try to parse
         # the document as each one in turn.
         is_html = not self.is_xml
+        if is_html:
+            self.processing_instruction_class = ProcessingInstruction
+        else:
+            self.processing_instruction_class = XMLProcessingInstruction
+
+        if isinstance(markup, str):
+            # We were given Unicode. Maybe lxml can parse Unicode on
+            # this system?
+            yield markup, None, document_declared_encoding, False
+
+        if isinstance(markup, str):
+            # No, apparently not. Convert the Unicode to UTF-8 and
+            # tell lxml to parse it as UTF-8.
+            yield (markup.encode("utf8"), "utf8",
+                   document_declared_encoding, False)
+
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        detector = EncodingDetector(markup, try_encodings, is_html)
+        detector = EncodingDetector(
+            markup, try_encodings, is_html, exclude_encodings)
         for encoding in detector.encodings:
             yield (detector.markup, encoding, document_declared_encoding, False)
 
     def feed(self, markup):
         if isinstance(markup, bytes):
             markup = BytesIO(markup)
-        elif isinstance(markup, unicode):
+        elif isinstance(markup, str):
             markup = StringIO(markup)
 
         # Call feed() at least once, even if the markup is empty,
@@ -117,30 +174,36 @@ class LXMLTreeBuilderForXML(TreeBuilder):
                 if len(data) != 0:
                     self.parser.feed(data)
             self.parser.close()
-        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
-            raise ParserRejectedMarkup(str(e))
+        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
+            raise ParserRejectedMarkup(e)
 
     def close(self):
-        self.nsmaps = [self.DEFAULT_NSMAPS]
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
 
     def start(self, name, attrs, nsmap={}):
         # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
         attrs = dict(attrs)
         nsprefix = None
         # Invert each namespace map as it comes in.
-        if len(self.nsmaps) > 1:
-            # There are no new namespaces for this tag, but
-            # non-default namespaces are in play, so we need a
-            # separate tag stack to know when they end.
-            self.nsmaps.append(None)
+        if len(nsmap) == 0 and len(self.nsmaps) > 1:
+                # There are no new namespaces for this tag, but
+                # non-default namespaces are in play, so we need a
+                # separate tag stack to know when they end.
+                self.nsmaps.append(None)
         elif len(nsmap) > 0:
             # A new namespace mapping has come into play.
-            inverted_nsmap = dict((value, key) for key, value in nsmap.items())
-            self.nsmaps.append(inverted_nsmap)
+
+            # First, Let the BeautifulSoup object know about it.
+            self._register_namespaces(nsmap)
+
+            # Then, add it to our running list of inverted namespace
+            # mappings.
+            self.nsmaps.append(_invert(nsmap))
+
             # Also treat the namespace mapping as a set of attributes on the
             # tag, so we can recreate it later.
             attrs = attrs.copy()
-            for prefix, namespace in nsmap.items():
+            for prefix, namespace in list(nsmap.items()):
                 attribute = NamespacedAttribute(
                     "xmlns", prefix, "http://www.w3.org/2000/xmlns/")
                 attrs[attribute] = namespace
@@ -149,7 +212,7 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         # from lxml with namespaces attached to their names, and
         # turn then into NamespacedAttribute objects.
         new_attrs = {}
-        for attr, value in attrs.items():
+        for attr, value in list(attrs.items()):
             namespace, attr = self._getNsTag(attr)
             if namespace is None:
                 new_attrs[attr] = value
@@ -189,7 +252,9 @@ class LXMLTreeBuilderForXML(TreeBuilder):
             self.nsmaps.pop()
 
     def pi(self, target, data):
-        pass
+        self.soup.endData()
+        self.soup.handle_data(target + ' ' + data)
+        self.soup.endData(self.processing_instruction_class)
 
     def data(self, content):
         self.soup.handle_data(content)
@@ -207,13 +272,17 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""
-        return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
+        return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
 
 
 class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
 
-    features = [LXML, HTML, FAST, PERMISSIVE]
+    NAME = LXML
+    ALTERNATE_NAMES = ["lxml-html"]
+
+    features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE]
     is_xml = False
+    processing_instruction_class = ProcessingInstruction
 
     def default_parser(self, encoding):
         return etree.HTMLParser
@@ -224,10 +293,10 @@ class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
             self.parser = self.parser_for(encoding)
             self.parser.feed(markup)
             self.parser.close()
-        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
-            raise ParserRejectedMarkup(str(e))
+        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
+            raise ParserRejectedMarkup(e)
 
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""
-        return u'<html><body>%s</body></html>' % fragment
+        return '<html><body>%s</body></html>' % fragment

lib/bs4/check_block.py

@@ -0,0 +1,4 @@
+import requests
+data = requests.get("https://www.crummy.com/").content
+from bs4 import _s
+data = [x for x in _s(data).block_text()]

lib/bs4/dammit.py

@@ -3,12 +3,15 @@
 
 This library converts a bytestream to Unicode through any means
 necessary. It is heavily based on code from Mark Pilgrim's Universal
-Feed Parser. It works best on XML and XML, but it does not rewrite the
+Feed Parser. It works best on XML and HTML, but it does not rewrite the
 XML or HTML to reflect a new encoding; that's the tree builder's job.
 """
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
 
 import codecs
-from htmlentitydefs import codepoint2name
+from future.moves.html.entities import codepoint2name
+from future.builtins import chr
 import re
 import logging
 import string
@@ -20,6 +23,8 @@ try:
     #  PyPI package: cchardet
     import cchardet
     def chardet_dammit(s):
+        if isinstance(s, str):
+            return None
         return cchardet.detect(s)['encoding']
 except ImportError:
     try:
@@ -28,6 +33,8 @@ except ImportError:
         #  PyPI package: chardet
         import chardet
         def chardet_dammit(s):
+            if isinstance(s, str):
+                return None
             return chardet.detect(s)['encoding']
         #import chardet.constants
         #chardet.constants._debug = 1
@@ -42,10 +49,19 @@ try:
 except ImportError:
     pass
 
-xml_encoding_re = re.compile(
-    '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
-html_meta_re = re.compile(
-    '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
+# Build bytestring and Unicode versions of regular expressions for finding
+# a declared encoding inside an XML or HTML document.
+xml_encoding = '^\s*<\\?.*encoding=[\'"](.*?)[\'"].*\\?>'
+html_meta = '<\\s*meta[^>]+charset\\s*=\\s*["\']?([^>]*?)[ /;\'">]'
+encoding_res = dict()
+encoding_res[bytes] = {
+    'html' : re.compile(html_meta.encode("ascii"), re.I),
+    'xml' : re.compile(xml_encoding.encode("ascii"), re.I),
+}
+encoding_res[str] = {
+    'html' : re.compile(html_meta, re.I),
+    'xml' : re.compile(xml_encoding, re.I)
+}
 
 class EntitySubstitution(object):
 
@@ -55,15 +71,24 @@ class EntitySubstitution(object):
         lookup = {}
         reverse_lookup = {}
         characters_for_re = []
-        for codepoint, name in list(codepoint2name.items()):
-            character = unichr(codepoint)
-            if codepoint != 34:
+
+        # &apos is an XHTML entity and an HTML 5, but not an HTML 4
+        # entity. We don't want to use it, but we want to recognize it on the way in.
+        #
+        # TODO: Ideally we would be able to recognize all HTML 5 named
+        # entities, but that's a little tricky.
+        extra = [(39, 'apos')]
+        for codepoint, name in list(codepoint2name.items()) + extra:
+            character = chr(codepoint)
+            if codepoint not in (34, 39):
                 # There's no point in turning the quotation mark into
-                # &quot;, unless it happens within an attribute value, which
-                # is handled elsewhere.
+                # &quot; or the single quote into &apos;, unless it
+                # happens within an attribute value, which is handled
+                # elsewhere.
                 characters_for_re.append(character)
                 lookup[character] = name
-            # But we do want to turn &quot; into the quotation mark.
+            # But we do want to recognize those entities on the way in and
+            # convert them to Unicode characters.
             reverse_lookup[name] = character
         re_definition = "[%s]" % "".join(characters_for_re)
         return lookup, reverse_lookup, re.compile(re_definition)
@@ -79,7 +104,7 @@ class EntitySubstitution(object):
         }
 
     BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
-                                           "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
+                                           "&(?!#\\d+;|#x[0-9a-fA-F]+;|\\w+;)"
                                            ")")
 
     AMPERSAND_OR_BRACKET = re.compile("([<>&])")
@@ -212,8 +237,11 @@ class EncodingDetector:
 
     5. Windows-1252.
     """
-    def __init__(self, markup, override_encodings=None, is_html=False):
+    def __init__(self, markup, override_encodings=None, is_html=False,
+                 exclude_encodings=None):
         self.override_encodings = override_encodings or []
+        exclude_encodings = exclude_encodings or []
+        self.exclude_encodings = set([x.lower() for x in exclude_encodings])
         self.chardet_encoding = None
         self.is_html = is_html
         self.declared_encoding = None
@@ -224,6 +252,8 @@ class EncodingDetector:
     def _usable(self, encoding, tried):
         if encoding is not None:
             encoding = encoding.lower()
+            if encoding in self.exclude_encodings:
+                return False
             if encoding not in tried:
                 tried.add(encoding)
                 return True
@@ -266,6 +296,9 @@ class EncodingDetector:
     def strip_byte_order_mark(cls, data):
         """If a byte-order mark is present, strip it and return the encoding it implies."""
         encoding = None
+        if isinstance(data, str):
+            # Unicode data cannot have a byte-order mark.
+            return data, encoding
         if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
                and (data[2:4] != '\x00\x00'):
             encoding = 'utf-16be'
@@ -300,14 +333,22 @@ class EncodingDetector:
             xml_endpos = 1024
             html_endpos = max(2048, int(len(markup) * 0.05))
 
+        if isinstance(markup, bytes):
+            res = encoding_res[bytes]
+        else:
+            res = encoding_res[str]
+
+        xml_re = res['xml']
+        html_re = res['html']
         declared_encoding = None
-        declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos)
+        declared_encoding_match = xml_re.search(markup, endpos=xml_endpos)
         if not declared_encoding_match and is_html:
-            declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos)
+            declared_encoding_match = html_re.search(markup, endpos=html_endpos)
         if declared_encoding_match is not None:
-            declared_encoding = declared_encoding_match.groups()[0].decode(
-                'ascii')
+            declared_encoding = declared_encoding_match.groups()[0]
         if declared_encoding:
+            if isinstance(declared_encoding, bytes):
+                declared_encoding = declared_encoding.decode('ascii', 'replace')
             return declared_encoding.lower()
         return None
 
@@ -331,18 +372,19 @@ class UnicodeDammit:
         ]
 
     def __init__(self, markup, override_encodings=[],
-                 smart_quotes_to=None, is_html=False):
+                 smart_quotes_to=None, is_html=False, exclude_encodings=[]):
         self.smart_quotes_to = smart_quotes_to
         self.tried_encodings = []
         self.contains_replacement_characters = False
         self.is_html = is_html
-
-        self.detector = EncodingDetector(markup, override_encodings, is_html)
+        self.log = logging.getLogger(__name__)
+        self.detector = EncodingDetector(
+            markup, override_encodings, is_html, exclude_encodings)
 
         # Short-circuit if the data is in Unicode to begin with.
-        if isinstance(markup, unicode) or markup == '':
+        if isinstance(markup, str) or markup == '':
             self.markup = markup
-            self.unicode_markup = unicode(markup)
+            self.unicode_markup = str(markup)
             self.original_encoding = None
             return
 
@@ -365,9 +407,10 @@ class UnicodeDammit:
                 if encoding != "ascii":
                     u = self._convert_from(encoding, "replace")
                 if u is not None:
-                    logging.warning(
+                    self.log.warning(
                             "Some characters could not be decoded, and were "
-                            "replaced with REPLACEMENT CHARACTER.")
+                            "replaced with REPLACEMENT CHARACTER."
+                    )
                     self.contains_replacement_characters = True
                     break
 
@@ -425,7 +468,7 @@ class UnicodeDammit:
     def _to_unicode(self, data, encoding, errors="strict"):
         '''Given a string and its encoding, decodes the string into Unicode.
         %encoding is a string recognized by encodings.aliases'''
-        return unicode(data, encoding, errors)
+        return str(data, encoding, errors)
 
     @property
     def declared_html_encoding(self):
@@ -723,7 +766,7 @@ class UnicodeDammit:
         0xde : b'\xc3\x9e',     # Þ
         0xdf : b'\xc3\x9f',     # ß
         0xe0 : b'\xc3\xa0',     # à
-        0xe1 : b'\xa1',     # á
+        0xe1 : b'\xa1',         # á
         0xe2 : b'\xc3\xa2',     # â
         0xe3 : b'\xc3\xa3',     # ã
         0xe4 : b'\xc3\xa4',     # ä

lib/bs4/diagnose.py

@@ -1,7 +1,11 @@
 """Diagnostic functions, mainly for use when doing tech support."""
+
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 import cProfile
-from StringIO import StringIO
-from HTMLParser import HTMLParser
+from io import StringIO
+from html.parser import HTMLParser
 import bs4
 from bs4 import BeautifulSoup, __version__
 from bs4.builder import builder_registry
@@ -17,8 +21,8 @@ import cProfile
 
 def diagnose(data):
     """Diagnostic suite for isolating common problems."""
-    print "Diagnostic running on Beautiful Soup %s" % __version__
-    print "Python version %s" % sys.version
+    print("Diagnostic running on Beautiful Soup %s" % __version__)
+    print("Python version %s" % sys.version)
 
     basic_parsers = ["html.parser", "html5lib", "lxml"]
     for name in basic_parsers:
@@ -27,44 +31,60 @@ def diagnose(data):
                 break
         else:
             basic_parsers.remove(name)
-            print (
+            print((
                 "I noticed that %s is not installed. Installing it may help." %
-                name)
+                name))
 
     if 'lxml' in basic_parsers:
-        basic_parsers.append(["lxml", "xml"])
-        from lxml import etree
-        print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))
+        basic_parsers.append("lxml-xml")
+        try:
+            from lxml import etree
+            print("Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)))
+        except ImportError as e:
+            print (
+                "lxml is not installed or couldn't be imported.")
+
 
     if 'html5lib' in basic_parsers:
-        import html5lib
-        print "Found html5lib version %s" % html5lib.__version__
+        try:
+            import html5lib
+            print("Found html5lib version %s" % html5lib.__version__)
+        except ImportError as e:
+            print (
+                "html5lib is not installed or couldn't be imported.")
 
     if hasattr(data, 'read'):
         data = data.read()
-    elif os.path.exists(data):
-        print '"%s" looks like a filename. Reading data from the file.' % data
-        data = open(data).read()
     elif data.startswith("http:") or data.startswith("https:"):
-        print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data
-        print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup."
+        print('"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data)
+        print("You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup.")
         return
-    print
+    else:
+        try:
+            if os.path.exists(data):
+                print('"%s" looks like a filename. Reading data from the file.' % data)
+                with open(data) as fp:
+                    data = fp.read()
+        except ValueError:
+            # This can happen on some platforms when the 'filename' is
+            # too long. Assume it's data and not a filename.
+            pass
+        print()
 
     for parser in basic_parsers:
-        print "Trying to parse your markup with %s" % parser
+        print("Trying to parse your markup with %s" % parser)
         success = False
         try:
-            soup = BeautifulSoup(data, parser)
+            soup = BeautifulSoup(data, features=parser)
             success = True
-        except Exception, e:
-            print "%s could not parse the markup." % parser
+        except Exception as e:
+            print("%s could not parse the markup." % parser)
             traceback.print_exc()
         if success:
-            print "Here's what %s did with the markup:" % parser
-            print soup.prettify()
+            print("Here's what %s did with the markup:" % parser)
+            print(soup.prettify())
 
-        print "-" * 80
+        print("-" * 80)
 
 def lxml_trace(data, html=True, **kwargs):
     """Print out the lxml events that occur during parsing.
@@ -74,7 +94,7 @@ def lxml_trace(data, html=True, **kwargs):
     """
     from lxml import etree
     for event, element in etree.iterparse(StringIO(data), html=html, **kwargs):
-        print("%s, %4s, %s" % (event, element.tag, element.text))
+        print(("%s, %4s, %s" % (event, element.tag, element.text)))
 
 class AnnouncingParser(HTMLParser):
     """Announces HTMLParser parse events, without doing anything else."""
@@ -135,7 +155,7 @@ def rword(length=5):
 def rsentence(length=4):
     "Generate a random sentence-like string."
     return " ".join(rword(random.randint(4,9)) for i in range(length))
-
+        
 def rdoc(num_elements=1000):
     """Randomly generate an invalid HTML document."""
     tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table']
@@ -156,10 +176,10 @@ def rdoc(num_elements=1000):
 
 def benchmark_parsers(num_elements=100000):
     """Very basic head-to-head performance benchmark."""
-    print "Comparative parser benchmark on Beautiful Soup %s" % __version__
+    print("Comparative parser benchmark on Beautiful Soup %s" % __version__)
     data = rdoc(num_elements)
-    print "Generated a large invalid HTML document (%d bytes)." % len(data)
-
+    print("Generated a large invalid HTML document (%d bytes)." % len(data))
+    
     for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
         success = False
         try:
@@ -167,24 +187,24 @@ def benchmark_parsers(num_elements=100000):
             soup = BeautifulSoup(data, parser)
             b = time.time()
             success = True
-        except Exception, e:
-            print "%s could not parse the markup." % parser
+        except Exception as e:
+            print("%s could not parse the markup." % parser)
             traceback.print_exc()
         if success:
-            print "BS4+%s parsed the markup in %.2fs." % (parser, b-a)
+            print("BS4+%s parsed the markup in %.2fs." % (parser, b-a))
 
     from lxml import etree
     a = time.time()
     etree.HTML(data)
     b = time.time()
-    print "Raw lxml parsed the markup in %.2fs." % (b-a)
+    print("Raw lxml parsed the markup in %.2fs." % (b-a))
 
     import html5lib
     parser = html5lib.HTMLParser()
     a = time.time()
     parser.parse(data)
     b = time.time()
-    print "Raw html5lib parsed the markup in %.2fs." % (b-a)
+    print("Raw html5lib parsed the markup in %.2fs." % (b-a))
 
 def profile(num_elements=100000, parser="lxml"):
 

lib/bs4/formatter.py

@@ -0,0 +1,99 @@
+from bs4.dammit import EntitySubstitution
+
+class Formatter(EntitySubstitution):
+    """Describes a strategy to use when outputting a parse tree to a string.
+
+    Some parts of this strategy come from the distinction between
+    HTML4, HTML5, and XML. Others are configurable by the user.
+    """
+    # Registries of XML and HTML formatters.
+    XML_FORMATTERS = {}
+    HTML_FORMATTERS = {}
+
+    HTML = 'html'
+    XML = 'xml'
+
+    HTML_DEFAULTS = dict(
+        cdata_containing_tags=set(["script", "style"]),
+    )
+
+    def _default(self, language, value, kwarg):
+        if value is not None:
+            return value
+        if language == self.XML:
+            return set()
+        return self.HTML_DEFAULTS[kwarg]
+
+    def __init__(
+            self, language=None, entity_substitution=None,
+            void_element_close_prefix='/', cdata_containing_tags=None,
+    ):
+        """
+
+        :param void_element_close_prefix: By default, represent void
+        elements as <tag/> rather than <tag>
+        """
+        self.language = language
+        self.entity_substitution = entity_substitution
+        self.void_element_close_prefix = void_element_close_prefix
+        self.cdata_containing_tags = self._default(
+            language, cdata_containing_tags, 'cdata_containing_tags'
+        )
+            
+    def substitute(self, ns):
+        """Process a string that needs to undergo entity substitution."""
+        if not self.entity_substitution:
+            return ns
+        from .element import NavigableString
+        if (isinstance(ns, NavigableString)
+            and ns.parent is not None
+            and ns.parent.name in self.cdata_containing_tags):
+            # Do nothing.
+            return ns
+        # Substitute.
+        return self.entity_substitution(ns)
+
+    def attribute_value(self, value):
+        """Process the value of an attribute."""
+        return self.substitute(value)
+    
+    def attributes(self, tag):
+        """Reorder a tag's attributes however you want."""
+        return sorted(tag.attrs.items())
+
+   
+class HTMLFormatter(Formatter):
+    REGISTRY = {}
+    def __init__(self, *args, **kwargs):
+        return super(HTMLFormatter, self).__init__(self.HTML, *args, **kwargs)
+
+    
+class XMLFormatter(Formatter):
+    REGISTRY = {}
+    def __init__(self, *args, **kwargs):
+        return super(XMLFormatter, self).__init__(self.XML, *args, **kwargs)
+
+
+# Set up aliases for the default formatters.
+HTMLFormatter.REGISTRY['html'] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html
+)
+HTMLFormatter.REGISTRY["html5"] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html,
+    void_element_close_prefix = None
+)
+HTMLFormatter.REGISTRY["minimal"] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_xml
+)
+HTMLFormatter.REGISTRY[None] = HTMLFormatter(
+    entity_substitution=None
+)
+XMLFormatter.REGISTRY["html"] =  XMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html
+)
+XMLFormatter.REGISTRY["minimal"] = XMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_xml
+)
+XMLFormatter.REGISTRY[None] = Formatter(
+    Formatter(Formatter.XML, entity_substitution=None)
+)

lib/bs4/testing.py

@@ -1,5 +1,10 @@
+# encoding: utf-8
 """Helper classes for tests."""
 
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
+import pickle
 import copy
 import functools
 import unittest
@@ -11,29 +16,66 @@ from bs4.element import (
     ContentMetaAttributeValue,
     Doctype,
     SoupStrainer,
+    Tag
 )
 
 from bs4.builder import HTMLParserTreeBuilder
 default_builder = HTMLParserTreeBuilder
 
+BAD_DOCUMENT = """A bare string
+<!DOCTYPE xsl:stylesheet SYSTEM "htmlent.dtd">
+<!DOCTYPE xsl:stylesheet PUBLIC "htmlent.dtd">
+<div><![CDATA[A CDATA section where it doesn't belong]]></div>
+<div><svg><![CDATA[HTML5 does allow CDATA sections in SVG]]></svg></div>
+<div>A <meta> tag</div>
+<div>A <br> tag that supposedly has contents.</br></div>
+<div>AT&T</div>
+<div><textarea>Within a textarea, markup like <b> tags and <&<&amp; should be treated as literal</textarea></div>
+<div><script>if (i < 2) { alert("<b>Markup within script tags should be treated as literal.</b>"); }</script></div>
+<div>This numeric entity is missing the final semicolon: <x t="pi&#241ata"></div>
+<div><a href="http://example.com/</a> that attribute value never got closed</div>
+<div><a href="foo</a>, </a><a href="bar">that attribute value was closed by the subsequent tag</a></div>
+<! This document starts with a bogus declaration ><div>a</div>
+<div>This document contains <!an incomplete declaration <div>(do you see it?)</div>
+<div>This document ends with <!an incomplete declaration
+<div><a style={height:21px;}>That attribute value was bogus</a></div>
+<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">The doctype is invalid because it contains extra whitespace
+<div><table><td nowrap>That boolean attribute had no value</td></table></div>
+<div>Here's a nonexistent entity: &#foo; (do you see it?)</div>
+<div>This document ends before the entity finishes: &gt
+<div><p>Paragraphs shouldn't contain block display elements, but this one does: <dl><dt>you see?</dt></p>
+<b b="20" a="1" b="10" a="2" a="3" a="4">Multiple values for the same attribute.</b>
+<div><table><tr><td>Here's a table</td></tr></table></div>
+<div><table id="1"><tr><td>Here's a nested table:<table id="2"><tr><td>foo</td></tr></table></td></div>
+<div>This tag contains nothing but whitespace: <b>    </b></div>
+<div><blockquote><p><b>This p tag is cut off by</blockquote></p>the end of the blockquote tag</div>
+<div><table><div>This table contains bare markup</div></table></div>
+<div><div id="1">\n <a href="link1">This link is never closed.\n</div>\n<div id="2">\n <div id="3">\n   <a href="link2">This link is closed.</a>\n  </div>\n</div></div>
+<div>This document contains a <!DOCTYPE surprise>surprise doctype</div>
+<div><a><B><Cd><EFG>Mixed case tags are folded to lowercase</efg></CD></b></A></div>
+<div><our\u2603>Tag name contains Unicode characters</our\u2603></div>
+<div><a \u2603="snowman">Attribute name contains Unicode characters</a></div>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+"""
+
 
 class SoupTest(unittest.TestCase):
 
     @property
     def default_builder(self):
-        return default_builder()
+        return default_builder
 
     def soup(self, markup, **kwargs):
         """Build a Beautiful Soup object from markup."""
         builder = kwargs.pop('builder', self.default_builder)
         return BeautifulSoup(markup, builder=builder, **kwargs)
 
-    def document_for(self, markup):
+    def document_for(self, markup, **kwargs):
         """Turn an HTML fragment into a document.
 
         The details depend on the builder.
         """
-        return self.default_builder.test_fragment_to_document(markup)
+        return self.default_builder(**kwargs).test_fragment_to_document(markup)
 
     def assertSoupEquals(self, to_parse, compare_parsed_to=None):
         builder = self.default_builder
@@ -43,6 +85,131 @@ class SoupTest(unittest.TestCase):
 
         self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
 
+    def assertConnectedness(self, element):
+        """Ensure that next_element and previous_element are properly
+        set for all descendants of the given element.
+        """
+        earlier = None
+        for e in element.descendants:
+            if earlier:
+                self.assertEqual(e, earlier.next_element)
+                self.assertEqual(earlier, e.previous_element)
+            earlier = e
+
+    def linkage_validator(self, el, _recursive_call=False):
+        """Ensure proper linkage throughout the document."""
+        descendant = None
+        # Document element should have no previous element or previous sibling.
+        # It also shouldn't have a next sibling.
+        if el.parent is None:
+            assert el.previous_element is None,\
+                "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format(
+                    el, el.previous_element, None
+                )
+            assert el.previous_sibling is None,\
+                "Bad previous_sibling\nNODE: {}\nPREV: {}\nEXPECTED: {}".format(
+                    el, el.previous_sibling, None
+                )
+            assert el.next_sibling is None,\
+                "Bad next_sibling\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format(
+                    el, el.next_sibling, None
+                )
+
+        idx = 0
+        child = None
+        last_child = None
+        last_idx = len(el.contents) - 1
+        for child in el.contents:
+            descendant = None
+
+            # Parent should link next element to their first child
+            # That child should have no previous sibling
+            if idx == 0:
+                if el.parent is not None:
+                    assert el.next_element is child,\
+                       "Bad next_element\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format(
+                            el, el.next_element, child
+                        )
+                    assert child.previous_element is el,\
+                       "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format(
+                            child, child.previous_element, el
+                        )
+                    assert child.previous_sibling is None,\
+                       "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED: {}".format(
+                            child, child.previous_sibling, None
+                        )
+
+            # If not the first child, previous index should link as sibling to this index
+            # Previous element should match the last index or the last bubbled up descendant
+            else:
+                assert child.previous_sibling is el.contents[idx - 1],\
+                    "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED {}".format(
+                        child, child.previous_sibling, el.contents[idx - 1]
+                    )
+                assert el.contents[idx - 1].next_sibling is child,\
+                    "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                        el.contents[idx - 1], el.contents[idx - 1].next_sibling, child
+                    )
+
+                if last_child is not None:
+                    assert child.previous_element is last_child,\
+                        "Bad previous_element\nNODE: {}\nPREV {}\nEXPECTED {}\nCONTENTS {}".format(
+                            child, child.previous_element, last_child, child.parent.contents
+                        )
+                    assert last_child.next_element is child,\
+                        "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                            last_child, last_child.next_element, child
+                        )
+
+            if isinstance(child, Tag) and child.contents:
+                descendant = self.linkage_validator(child, True)
+                # A bubbled up descendant should have no next siblings
+                assert descendant.next_sibling is None,\
+                    "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                        descendant, descendant.next_sibling, None
+                    )
+
+            # Mark last child as either the bubbled up descendant or the current child
+            if descendant is not None:
+                last_child = descendant
+            else:
+                last_child = child
+
+            # If last child, there are non next siblings
+            if idx == last_idx:
+                assert child.next_sibling is None,\
+                    "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                        child, child.next_sibling, None
+                    )
+            idx += 1
+
+        child = descendant if descendant is not None else child
+        if child is None:
+            child = el
+
+        if not _recursive_call and child is not None:
+            target = el
+            while True:
+                if target is None:
+                    assert child.next_element is None, \
+                        "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                            child, child.next_element, None
+                        )
+                    break
+                elif target.next_sibling is not None:
+                    assert child.next_element is target.next_sibling, \
+                        "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                            child, child.next_element, target.next_sibling
+                        )
+                    break
+                target = target.parent
+
+            # We are done, so nothing to return
+            return None
+        else:
+            # Return the child to the recursive caller
+            return child
+
 
 class HTMLTreeBuilderSmokeTest(object):
 
@@ -54,6 +221,27 @@ class HTMLTreeBuilderSmokeTest(object):
     markup in these tests, there's not much room for interpretation.
     """
 
+    def test_empty_element_tags(self):
+        """Verify that all HTML4 and HTML5 empty element (aka void element) tags
+        are handled correctly.
+        """
+        for name in [
+                'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
+                'spacer', 'frame'
+        ]:
+            soup = self.soup("")
+            new_tag = soup.new_tag(name)
+            self.assertEqual(True, new_tag.is_empty_element)
+
+    def test_pickle_and_unpickle_identity(self):
+        # Pickling a tree, then unpickling it, yields a tree identical
+        # to the original.
+        tree = self.soup("<a><b>foo</a>")
+        dumped = pickle.dumps(tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertEqual(loaded.__class__, BeautifulSoup)
+        self.assertEqual(loaded.decode(), tree.decode())
+
     def assertDoctypeHandled(self, doctype_fragment):
         """Assert that a given doctype string is handled correctly."""
         doctype_str, soup = self._document_with_doctype(doctype_fragment)
@@ -114,6 +302,27 @@ class HTMLTreeBuilderSmokeTest(object):
             soup.encode("utf-8").replace(b"\n", b""),
             markup.replace(b"\n", b""))
 
+    def test_namespaced_html(self):
+        """When a namespaced XML document is parsed as HTML it should
+        be treated as HTML with weird tag names.
+        """
+        markup = b"""<ns1:foo>content</ns1:foo><ns1:foo/><ns2:foo/>"""
+        soup = self.soup(markup)
+        self.assertEqual(2, len(soup.find_all("ns1:foo")))
+        
+    def test_processing_instruction(self):
+        # We test both Unicode and bytestring to verify that
+        # process_markup correctly sets processing_instruction_class
+        # even when the markup is already Unicode and there is no
+        # need to process anything.
+        markup = """<?PITarget PIContent?>"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.decode())
+
+        markup = b"""<?PITarget PIContent?>"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode("utf8"))
+
     def test_deepcopy(self):
         """Make sure you can copy the tree builder.
 
@@ -155,6 +364,23 @@ class HTMLTreeBuilderSmokeTest(object):
     def test_nested_formatting_elements(self):
         self.assertSoupEquals("<em><em></em></em>")
 
+    def test_double_head(self):
+        html = '''<!DOCTYPE html>
+<html>
+<head>
+<title>Ordinary HEAD element test</title>
+</head>
+<script type="text/javascript">
+alert("Help!");
+</script>
+<body>
+Hello, world!
+</body>
+</html>
+'''
+        soup = self.soup(html)
+        self.assertEqual("text/javascript", soup.find('script')['type'])
+
     def test_comment(self):
         # Comments are represented as Comment objects.
         markup = "<p>foo<!--foobar-->baz</p>"
@@ -171,9 +397,22 @@ class HTMLTreeBuilderSmokeTest(object):
         self.assertEqual(comment, baz.previous_element)
 
     def test_preserved_whitespace_in_pre_and_textarea(self):
-        """Whitespace must be preserved in <pre> and <textarea> tags."""
-        self.assertSoupEquals("<pre>   </pre>")
-        self.assertSoupEquals("<textarea> woo  </textarea>")
+        """Whitespace must be preserved in <pre> and <textarea> tags,
+        even if that would mean not prettifying the markup.
+        """
+        pre_markup = "<pre>   </pre>"
+        textarea_markup = "<textarea> woo\nwoo  </textarea>"
+        self.assertSoupEquals(pre_markup)
+        self.assertSoupEquals(textarea_markup)
+
+        soup = self.soup(pre_markup)
+        self.assertEqual(soup.pre.prettify(), pre_markup)
+
+        soup = self.soup(textarea_markup)
+        self.assertEqual(soup.textarea.prettify(), textarea_markup)
+
+        soup = self.soup("<textarea></textarea>")
+        self.assertEqual(soup.textarea.prettify(), "<textarea></textarea>")
 
     def test_nested_inline_elements(self):
         """Inline elements can be nested indefinitely."""
@@ -213,6 +452,18 @@ class HTMLTreeBuilderSmokeTest(object):
             "<tbody><tr><td>Bar</td></tr></tbody>"
             "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
 
+    def test_multivalued_attribute_with_whitespace(self):
+        # Whitespace separating the values of a multi-valued attribute
+        # should be ignored.
+
+        markup = '<div class=" foo bar	 "></a>'
+        soup = self.soup(markup)
+        self.assertEqual(['foo', 'bar'], soup.div['class'])
+
+        # If you search by the literal name of the class it's like the whitespace
+        # wasn't there.
+        self.assertEqual(soup.div, soup.find('div', class_="foo bar"))
+        
     def test_deeply_nested_multivalued_attribute(self):
         # html5lib can set the attributes of the same tag many times
         # as it rearranges the tree. This has caused problems with
@@ -221,18 +472,52 @@ class HTMLTreeBuilderSmokeTest(object):
         soup = self.soup(markup)
         self.assertEqual(["css"], soup.div.div['class'])
 
+    def test_multivalued_attribute_on_html(self):
+        # html5lib uses a different API to set the attributes ot the
+        # <html> tag. This has caused problems with multivalued
+        # attributes.
+        markup = '<html class="a b"></html>'
+        soup = self.soup(markup)
+        self.assertEqual(["a", "b"], soup.html['class'])
+
     def test_angle_brackets_in_attribute_values_are_escaped(self):
         self.assertSoupEquals('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')
 
+    def test_strings_resembling_character_entity_references(self):
+        # "&T" and "&p" look like incomplete character entities, but they are
+        # not.
+        self.assertSoupEquals(
+            "<p>&bull; AT&T is in the s&p 500</p>",
+            "<p>\u2022 AT&amp;T is in the s&amp;p 500</p>"
+        )
+
+    def test_apos_entity(self):
+        self.assertSoupEquals(
+            "<p>Bob&apos;s Bar</p>",
+            "<p>Bob's Bar</p>",
+        )
+        
+    def test_entities_in_foreign_document_encoding(self):
+        # &#147; and &#148; are invalid numeric entities referencing
+        # Windows-1252 characters. &#45; references a character common
+        # to Windows-1252 and Unicode, and &#9731; references a
+        # character only found in Unicode.
+        #
+        # All of these entities should be converted to Unicode
+        # characters.
+        markup = "<p>&#147;Hello&#148; &#45;&#9731;</p>"
+        soup = self.soup(markup)
+        self.assertEqual("“Hello” -☃", soup.p.string)
+        
     def test_entities_in_attributes_converted_to_unicode(self):
-        expect = u'<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
+        expect = '<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
         self.assertSoupEquals('<p id="pi&#241;ata"></p>', expect)
         self.assertSoupEquals('<p id="pi&#xf1;ata"></p>', expect)
         self.assertSoupEquals('<p id="pi&#Xf1;ata"></p>', expect)
         self.assertSoupEquals('<p id="pi&ntilde;ata"></p>', expect)
 
     def test_entities_in_text_converted_to_unicode(self):
-        expect = u'<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
+        expect = '<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
         self.assertSoupEquals("<p>pi&#241;ata</p>", expect)
         self.assertSoupEquals("<p>pi&#xf1;ata</p>", expect)
         self.assertSoupEquals("<p>pi&#Xf1;ata</p>", expect)
@@ -243,16 +528,52 @@ class HTMLTreeBuilderSmokeTest(object):
                               '<p>I said "good day!"</p>')
 
     def test_out_of_range_entity(self):
-        expect = u"\N{REPLACEMENT CHARACTER}"
+        expect = "\N{REPLACEMENT CHARACTER}"
         self.assertSoupEquals("&#10000000000000;", expect)
         self.assertSoupEquals("&#x10000000000000;", expect)
         self.assertSoupEquals("&#1000000000;", expect)
-
+        
     def test_multipart_strings(self):
         "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
         soup = self.soup("<html><h2>\nfoo</h2><p></p></html>")
         self.assertEqual("p", soup.h2.string.next_element.name)
         self.assertEqual("p", soup.p.name)
+        self.assertConnectedness(soup)
+
+    def test_empty_element_tags(self):
+        """Verify consistent handling of empty-element tags,
+        no matter how they come in through the markup.
+        """
+        self.assertSoupEquals('<br/><br/><br/>', "<br/><br/><br/>")
+        self.assertSoupEquals('<br /><br /><br />', "<br/><br/><br/>")
+        
+    def test_head_tag_between_head_and_body(self):
+        "Prevent recurrence of a bug in the html5lib treebuilder."
+        content = """<html><head></head>
+  <link></link>
+  <body>foo</body>
+</html>
+"""
+        soup = self.soup(content)
+        self.assertNotEqual(None, soup.html.body)
+        self.assertConnectedness(soup)
+
+    def test_multiple_copies_of_a_tag(self):
+        "Prevent recurrence of a bug in the html5lib treebuilder."
+        content = """<!DOCTYPE html>
+<html>
+ <body>
+   <article id="a" >
+   <div><a href="1"></div>
+   <footer>
+     <a href="2"></a>
+   </footer>
+  </article>
+  </body>
+</html>
+"""
+        soup = self.soup(content)
+        self.assertConnectedness(soup.article)
 
     def test_basic_namespaces(self):
         """Parsers don't need to *understand* namespaces, but at the
@@ -285,9 +606,9 @@ class HTMLTreeBuilderSmokeTest(object):
         # A seemingly innocuous document... but it's in Unicode! And
         # it contains characters that can't be represented in the
         # encoding found in the  declaration! The horror!
-        markup = u'<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>'
+        markup = '<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>'
         soup = self.soup(markup)
-        self.assertEqual(u'Sacr\xe9 bleu!', soup.body.string)
+        self.assertEqual('Sacr\xe9 bleu!', soup.body.string)
 
     def test_soupstrainer(self):
         """Parsers should be able to work with SoupStrainers."""
@@ -327,7 +648,7 @@ class HTMLTreeBuilderSmokeTest(object):
         # Both XML and HTML entities are converted to Unicode characters
         # during parsing.
         text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
-        expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
+        expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
         self.assertSoupEquals(text, expected)
 
     def test_smart_quotes_converted_on_the_way_in(self):
@@ -337,15 +658,15 @@ class HTMLTreeBuilderSmokeTest(object):
         soup = self.soup(quote)
         self.assertEqual(
             soup.p.string,
-            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
+            "\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
 
     def test_non_breaking_spaces_converted_on_the_way_in(self):
         soup = self.soup("<a>&nbsp;&nbsp;</a>")
-        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
+        self.assertEqual(soup.a.string, "\N{NO-BREAK SPACE}" * 2)
 
     def test_entities_converted_on_the_way_out(self):
         text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
-        expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
+        expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
         soup = self.soup(text)
         self.assertEqual(soup.p.encode("utf-8"), expected)
 
@@ -354,7 +675,7 @@ class HTMLTreeBuilderSmokeTest(object):
         # easy-to-understand document.
 
         # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
-        unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
+        unicode_html = '<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
 
         # That's because we're going to encode it into ISO-Latin-1, and use
         # that to test.
@@ -399,7 +720,9 @@ class HTMLTreeBuilderSmokeTest(object):
         hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
         soup = self.soup(
             hebrew_document, from_encoding="iso8859-8")
-        self.assertEqual(soup.original_encoding, 'iso8859-8')
+        # Some tree builders call it iso8859-8, others call it iso-8859-9.
+        # That's not a difference we really care about.
+        assert soup.original_encoding in ('iso8859-8', 'iso-8859-8')
         self.assertEqual(
             soup.encode('utf-8'),
             hebrew_document.decode("iso8859-8").encode("utf-8"))
@@ -461,13 +784,39 @@ class HTMLTreeBuilderSmokeTest(object):
         data.a['foo'] = 'bar'
         self.assertEqual('<a foo="bar">text</a>', data.a.decode())
 
+    def test_worst_case(self):
+        """Test the worst case (currently) for linking issues."""
+
+        soup = self.soup(BAD_DOCUMENT)
+        self.linkage_validator(soup)
+
+
 class XMLTreeBuilderSmokeTest(object):
 
+    def test_pickle_and_unpickle_identity(self):
+        # Pickling a tree, then unpickling it, yields a tree identical
+        # to the original.
+        tree = self.soup("<a><b>foo</a>")
+        dumped = pickle.dumps(tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertEqual(loaded.__class__, BeautifulSoup)
+        self.assertEqual(loaded.decode(), tree.decode())
+
     def test_docstring_generated(self):
         soup = self.soup("<root/>")
         self.assertEqual(
             soup.encode(), b'<?xml version="1.0" encoding="utf-8"?>\n<root/>')
 
+    def test_xml_declaration(self):
+        markup = b"""<?xml version="1.0" encoding="utf8"?>\n<foo/>"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode("utf8"))
+
+    def test_processing_instruction(self):
+        markup = b"""<?xml version="1.0" encoding="utf8"?>\n<?PITarget PIContent?>"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode("utf8"))
+
     def test_real_xhtml_document(self):
         """A real XHTML document should come out *exactly* the same as it went in."""
         markup = b"""<?xml version="1.0" encoding="utf-8"?>
@@ -480,12 +829,23 @@ class XMLTreeBuilderSmokeTest(object):
         self.assertEqual(
             soup.encode("utf-8"), markup)
 
+    def test_nested_namespaces(self):
+        doc = b"""<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<parent xmlns="http://ns1/">
+<child xmlns="http://ns2/" xmlns:ns3="http://ns3/">
+<grandchild ns3:attr="value" xmlns="http://ns4/"/>
+</child>
+</parent>"""
+        soup = self.soup(doc)
+        self.assertEqual(doc, soup.encode())
+        
     def test_formatter_processes_script_tag_for_xml_documents(self):
         doc = """
   <script type="text/javascript">
   </script>
 """
-        soup = BeautifulSoup(doc, "xml")
+        soup = BeautifulSoup(doc, "lxml-xml")
         # lxml would have stripped this while parsing, but we can add
         # it later.
         soup.script.string = 'console.log("< < hey > > ");'
@@ -493,15 +853,15 @@ class XMLTreeBuilderSmokeTest(object):
         self.assertTrue(b"&lt; &lt; hey &gt; &gt;" in encoded)
 
     def test_can_parse_unicode_document(self):
-        markup = u'<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
+        markup = '<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
         soup = self.soup(markup)
-        self.assertEqual(u'Sacr\xe9 bleu!', soup.root.string)
+        self.assertEqual('Sacr\xe9 bleu!', soup.root.string)
 
     def test_popping_namespaced_tag(self):
         markup = '<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>'
         soup = self.soup(markup)
         self.assertEqual(
-            unicode(soup.rss), markup)
+            str(soup.rss), markup)
 
     def test_docstring_includes_correct_encoding(self):
         soup = self.soup("<root/>")
@@ -532,17 +892,57 @@ class XMLTreeBuilderSmokeTest(object):
     def test_closing_namespaced_tag(self):
         markup = '<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>'
         soup = self.soup(markup)
-        self.assertEqual(unicode(soup.p), markup)
+        self.assertEqual(str(soup.p), markup)
 
     def test_namespaced_attributes(self):
         markup = '<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>'
         soup = self.soup(markup)
-        self.assertEqual(unicode(soup.foo), markup)
+        self.assertEqual(str(soup.foo), markup)
 
     def test_namespaced_attributes_xml_namespace(self):
         markup = '<foo xml:lang="fr">bar</foo>'
         soup = self.soup(markup)
-        self.assertEqual(unicode(soup.foo), markup)
+        self.assertEqual(str(soup.foo), markup)
+
+    def test_find_by_prefixed_name(self):
+        doc = """<?xml version="1.0" encoding="utf-8"?>
+<Document xmlns="http://example.com/ns0"
+    xmlns:ns1="http://example.com/ns1"
+    xmlns:ns2="http://example.com/ns2"
+    <ns1:tag>foo</ns1:tag>
+    <ns1:tag>bar</ns1:tag>
+    <ns2:tag key="value">baz</ns2:tag>
+</Document>
+"""
+        soup = self.soup(doc)
+
+        # There are three <tag> tags.
+        self.assertEqual(3, len(soup.find_all('tag')))
+
+        # But two of them are ns1:tag and one of them is ns2:tag.
+        self.assertEqual(2, len(soup.find_all('ns1:tag')))
+        self.assertEqual(1, len(soup.find_all('ns2:tag')))
+        
+        self.assertEqual(1, len(soup.find_all('ns2:tag', key='value')))
+        self.assertEqual(3, len(soup.find_all(['ns1:tag', 'ns2:tag'])))
+        
+    def test_copy_tag_preserves_namespace(self):
+        xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<w:document xmlns:w="http://example.com/ns0"/>"""
+    
+        soup = self.soup(xml)
+        tag = soup.document
+        duplicate = copy.copy(tag)
+
+        # The two tags have the same namespace prefix.
+        self.assertEqual(tag.prefix, duplicate.prefix)
+
+    def test_worst_case(self):
+        """Test the worst case (currently) for linking issues."""
+
+        soup = self.soup(BAD_DOCUMENT)
+        self.linkage_validator(soup)
+
 
 class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
     """Smoke test for a tree builder that supports HTML5."""

lib/bs4/tests/__init__.py

@@ -0,0 +1 @@
+"The beautifulsoup tests."

lib/bs4/tests/test_builder_registry.py

@@ -0,0 +1,147 @@
+"""Tests of the builder registry."""
+
+import unittest
+import warnings
+
+from bs4 import BeautifulSoup
+from bs4.builder import (
+    builder_registry as registry,
+    HTMLParserTreeBuilder,
+    TreeBuilderRegistry,
+)
+
+try:
+    from bs4.builder import HTML5TreeBuilder
+    HTML5LIB_PRESENT = True
+except ImportError:
+    HTML5LIB_PRESENT = False
+
+try:
+    from bs4.builder import (
+        LXMLTreeBuilderForXML,
+        LXMLTreeBuilder,
+        )
+    LXML_PRESENT = True
+except ImportError:
+    LXML_PRESENT = False
+
+
+class BuiltInRegistryTest(unittest.TestCase):
+    """Test the built-in registry with the default builders registered."""
+
+    def test_combination(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('fast', 'html'),
+                             LXMLTreeBuilder)
+
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('permissive', 'xml'),
+                             LXMLTreeBuilderForXML)
+        self.assertEqual(registry.lookup('strict', 'html'),
+                          HTMLParserTreeBuilder)
+        if HTML5LIB_PRESENT:
+            self.assertEqual(registry.lookup('html5lib', 'html'),
+                              HTML5TreeBuilder)
+
+    def test_lookup_by_markup_type(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('html'), LXMLTreeBuilder)
+            self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML)
+        else:
+            self.assertEqual(registry.lookup('xml'), None)
+            if HTML5LIB_PRESENT:
+                self.assertEqual(registry.lookup('html'), HTML5TreeBuilder)
+            else:
+                self.assertEqual(registry.lookup('html'), HTMLParserTreeBuilder)
+
+    def test_named_library(self):
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('lxml', 'xml'),
+                             LXMLTreeBuilderForXML)
+            self.assertEqual(registry.lookup('lxml', 'html'),
+                             LXMLTreeBuilder)
+        if HTML5LIB_PRESENT:
+            self.assertEqual(registry.lookup('html5lib'),
+                              HTML5TreeBuilder)
+
+        self.assertEqual(registry.lookup('html.parser'),
+                          HTMLParserTreeBuilder)
+
+    def test_beautifulsoup_constructor_does_lookup(self):
+
+        with warnings.catch_warnings(record=True) as w:
+            # This will create a warning about not explicitly
+            # specifying a parser, but we'll ignore it.
+
+            # You can pass in a string.
+            BeautifulSoup("", features="html")
+            # Or a list of strings.
+            BeautifulSoup("", features=["html", "fast"])
+
+        # You'll get an exception if BS can't find an appropriate
+        # builder.
+        self.assertRaises(ValueError, BeautifulSoup,
+                          "", features="no-such-feature")
+
+class RegistryTest(unittest.TestCase):
+    """Test the TreeBuilderRegistry class in general."""
+
+    def setUp(self):
+        self.registry = TreeBuilderRegistry()
+
+    def builder_for_features(self, *feature_list):
+        cls = type('Builder_' + '_'.join(feature_list),
+                   (object,), {'features' : feature_list})
+
+        self.registry.register(cls)
+        return cls
+
+    def test_register_with_no_features(self):
+        builder = self.builder_for_features()
+
+        # Since the builder advertises no features, you can't find it
+        # by looking up features.
+        self.assertEqual(self.registry.lookup('foo'), None)
+
+        # But you can find it by doing a lookup with no features, if
+        # this happens to be the only registered builder.
+        self.assertEqual(self.registry.lookup(), builder)
+
+    def test_register_with_features_makes_lookup_succeed(self):
+        builder = self.builder_for_features('foo', 'bar')
+        self.assertEqual(self.registry.lookup('foo'), builder)
+        self.assertEqual(self.registry.lookup('bar'), builder)
+
+    def test_lookup_fails_when_no_builder_implements_feature(self):
+        builder = self.builder_for_features('foo', 'bar')
+        self.assertEqual(self.registry.lookup('baz'), None)
+
+    def test_lookup_gets_most_recent_registration_when_no_feature_specified(self):
+        builder1 = self.builder_for_features('foo')
+        builder2 = self.builder_for_features('bar')
+        self.assertEqual(self.registry.lookup(), builder2)
+
+    def test_lookup_fails_when_no_tree_builders_registered(self):
+        self.assertEqual(self.registry.lookup(), None)
+
+    def test_lookup_gets_most_recent_builder_supporting_all_features(self):
+        has_one = self.builder_for_features('foo')
+        has_the_other = self.builder_for_features('bar')
+        has_both_early = self.builder_for_features('foo', 'bar', 'baz')
+        has_both_late = self.builder_for_features('foo', 'bar', 'quux')
+        lacks_one = self.builder_for_features('bar')
+        has_the_other = self.builder_for_features('foo')
+
+        # There are two builders featuring 'foo' and 'bar', but
+        # the one that also features 'quux' was registered later.
+        self.assertEqual(self.registry.lookup('foo', 'bar'),
+                          has_both_late)
+
+        # There is only one builder featuring 'foo', 'bar', and 'baz'.
+        self.assertEqual(self.registry.lookup('foo', 'bar', 'baz'),
+                          has_both_early)
+
+    def test_lookup_fails_when_cannot_reconcile_requested_features(self):
+        builder1 = self.builder_for_features('foo', 'bar')
+        builder2 = self.builder_for_features('foo', 'baz')
+        self.assertEqual(self.registry.lookup('bar', 'baz'), None)

lib/bs4/tests/test_docs.py

@@ -0,0 +1,36 @@
+"Test harness for doctests."
+
+# pylint: disable-msg=E0611,W0142
+
+__metaclass__ = type
+__all__ = [
+    'additional_tests',
+    ]
+
+import atexit
+import doctest
+import os
+#from pkg_resources import (
+#    resource_filename, resource_exists, resource_listdir, cleanup_resources)
+import unittest
+
+DOCTEST_FLAGS = (
+    doctest.ELLIPSIS |
+    doctest.NORMALIZE_WHITESPACE |
+    doctest.REPORT_NDIFF)
+
+
+# def additional_tests():
+#     "Run the doc tests (README.txt and docs/*, if any exist)"
+#     doctest_files = [
+#         os.path.abspath(resource_filename('bs4', 'README.txt'))]
+#     if resource_exists('bs4', 'docs'):
+#         for name in resource_listdir('bs4', 'docs'):
+#             if name.endswith('.txt'):
+#                 doctest_files.append(
+#                     os.path.abspath(
+#                         resource_filename('bs4', 'docs/%s' % name)))
+#     kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
+#     atexit.register(cleanup_resources)
+#     return unittest.TestSuite((
+#         doctest.DocFileSuite(*doctest_files, **kwargs)))

lib/bs4/tests/test_html5lib.py

@@ -0,0 +1,184 @@
+"""Tests to ensure that the html5lib tree builder generates good trees."""
+
+import warnings
+
+try:
+    from bs4.builder import HTML5TreeBuilder
+    HTML5LIB_PRESENT = True
+except ImportError as e:
+    HTML5LIB_PRESENT = False
+from bs4.element import SoupStrainer
+from bs4.testing import (
+    HTML5TreeBuilderSmokeTest,
+    SoupTest,
+    skipIf,
+)
+
+@skipIf(
+    not HTML5LIB_PRESENT,
+    "html5lib seems not to be present, not testing its tree builder.")
+class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest):
+    """See ``HTML5TreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return HTML5TreeBuilder
+
+    def test_soupstrainer(self):
+        # The html5lib tree builder does not support SoupStrainers.
+        strainer = SoupStrainer("b")
+        markup = "<p>A <b>bold</b> statement.</p>"
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup(markup, parse_only=strainer)
+        self.assertEqual(
+            soup.decode(), self.document_for(markup))
+
+        self.assertTrue(
+            "the html5lib tree builder doesn't support parse_only" in
+            str(w[0].message))
+
+    def test_correctly_nested_tables(self):
+        """html5lib inserts <tbody> tags where other parsers don't."""
+        markup = ('<table id="1">'
+                  '<tr>'
+                  "<td>Here's another table:"
+                  '<table id="2">'
+                  '<tr><td>foo</td></tr>'
+                  '</table></td>')
+
+        self.assertSoupEquals(
+            markup,
+            '<table id="1"><tbody><tr><td>Here\'s another table:'
+            '<table id="2"><tbody><tr><td>foo</td></tr></tbody></table>'
+            '</td></tr></tbody></table>')
+
+        self.assertSoupEquals(
+            "<table><thead><tr><td>Foo</td></tr></thead>"
+            "<tbody><tr><td>Bar</td></tr></tbody>"
+            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
+
+    def test_xml_declaration_followed_by_doctype(self):
+        markup = '''<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html>
+<html>
+  <head>
+  </head>
+  <body>
+   <p>foo</p>
+  </body>
+</html>'''
+        soup = self.soup(markup)
+        # Verify that we can reach the <p> tag; this means the tree is connected.
+        self.assertEqual(b"<p>foo</p>", soup.p.encode())
+
+    def test_reparented_markup(self):
+        markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>'
+        soup = self.soup(markup)
+        self.assertEqual("<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p></body>", soup.body.decode())
+        self.assertEqual(2, len(soup.find_all('p')))
+
+
+    def test_reparented_markup_ends_with_whitespace(self):
+        markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>\n'
+        soup = self.soup(markup)
+        self.assertEqual("<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p>\n</body>", soup.body.decode())
+        self.assertEqual(2, len(soup.find_all('p')))
+
+    def test_reparented_markup_containing_identical_whitespace_nodes(self):
+        """Verify that we keep the two whitespace nodes in this
+        document distinct when reparenting the adjacent <tbody> tags.
+        """
+        markup = '<table> <tbody><tbody><ims></tbody> </table>'
+        soup = self.soup(markup)
+        space1, space2 = soup.find_all(string=' ')
+        tbody1, tbody2 = soup.find_all('tbody')
+        assert space1.next_element is tbody1
+        assert tbody2.next_element is space2
+
+    def test_reparented_markup_containing_children(self):
+        markup = '<div><a>aftermath<p><noscript>target</noscript>aftermath</a></p></div>'
+        soup = self.soup(markup)
+        noscript = soup.noscript
+        self.assertEqual("target", noscript.next_element)
+        target = soup.find(string='target')
+
+        # The 'aftermath' string was duplicated; we want the second one.
+        final_aftermath = soup.find_all(string='aftermath')[-1]
+
+        # The <noscript> tag was moved beneath a copy of the <a> tag,
+        # but the 'target' string within is still connected to the
+        # (second) 'aftermath' string.
+        self.assertEqual(final_aftermath, target.next_element)
+        self.assertEqual(target, final_aftermath.previous_element)
+        
+    def test_processing_instruction(self):
+        """Processing instructions become comments."""
+        markup = b"""<?PITarget PIContent?>"""
+        soup = self.soup(markup)
+        assert str(soup).startswith("<!--?PITarget PIContent?-->")
+
+    def test_cloned_multivalue_node(self):
+        markup = b"""<a class="my_class"><p></a>"""
+        soup = self.soup(markup)
+        a1, a2 = soup.find_all('a')
+        self.assertEqual(a1, a2)
+        assert a1 is not a2
+
+    def test_foster_parenting(self):
+        markup = b"""<table><td></tbody>A"""
+        soup = self.soup(markup)
+        self.assertEqual("<body>A<table><tbody><tr><td></td></tr></tbody></table></body>", soup.body.decode())
+
+    def test_extraction(self):
+        """
+        Test that extraction does not destroy the tree.
+
+        https://bugs.launchpad.net/beautifulsoup/+bug/1782928
+        """
+
+        markup = """
+<html><head></head>
+<style>
+</style><script></script><body><p>hello</p></body></html>
+"""
+        soup = self.soup(markup)
+        [s.extract() for s in soup('script')]
+        [s.extract() for s in soup('style')]
+
+        self.assertEqual(len(soup.find_all("p")), 1)
+
+    def test_empty_comment(self):
+        """
+        Test that empty comment does not break structure.
+
+        https://bugs.launchpad.net/beautifulsoup/+bug/1806598
+        """
+
+        markup = """
+<html>
+<body>
+<form>
+<!----><input type="text">
+</form>
+</body>
+</html>
+"""
+        soup = self.soup(markup)
+        inputs = []
+        for form in soup.find_all('form'):
+            inputs.extend(form.find_all('input'))
+        self.assertEqual(len(inputs), 1)
+
+    def test_tracking_line_numbers(self):
+        # The html.parser TreeBuilder keeps track of line number and
+        # position of each element.
+        markup = "\n   <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>"
+        soup = self.soup(markup)
+        self.assertEqual(2, soup.p.sourceline)
+        self.assertEqual(5, soup.p.sourcepos)
+        self.assertEqual("sourceline", soup.p.find('sourceline').name)
+
+        # You can deactivate this behavior.
+        soup = self.soup(markup, store_line_numbers=False)
+        self.assertEqual("sourceline", soup.p.sourceline.name)
+        self.assertEqual("sourcepos", soup.p.sourcepos.name)

lib/bs4/tests/test_htmlparser.py

@@ -0,0 +1,61 @@
+"""Tests to ensure that the html.parser tree builder generates good
+trees."""
+
+from pdb import set_trace
+import pickle
+from bs4.testing import SoupTest, HTMLTreeBuilderSmokeTest
+from bs4.builder import HTMLParserTreeBuilder
+from bs4.builder._htmlparser import BeautifulSoupHTMLParser
+
+class HTMLParserTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
+
+    default_builder = HTMLParserTreeBuilder
+
+    def test_namespaced_system_doctype(self):
+        # html.parser can't handle namespaced doctypes, so skip this one.
+        pass
+
+    def test_namespaced_public_doctype(self):
+        # html.parser can't handle namespaced doctypes, so skip this one.
+        pass
+
+    def test_builder_is_pickled(self):
+        """Unlike most tree builders, HTMLParserTreeBuilder and will
+        be restored after pickling.
+        """
+        tree = self.soup("<a><b>foo</a>")
+        dumped = pickle.dumps(tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertTrue(isinstance(loaded.builder, type(tree.builder)))
+
+    def test_redundant_empty_element_closing_tags(self):
+        self.assertSoupEquals('<br></br><br></br><br></br>', "<br/><br/><br/>")
+        self.assertSoupEquals('</br></br></br>', "")
+
+    def test_empty_element(self):
+        # This verifies that any buffered data present when the parser
+        # finishes working is handled.
+        self.assertSoupEquals("foo &# bar", "foo &amp;# bar")
+
+    def test_tracking_line_numbers(self):
+        # The html.parser TreeBuilder keeps track of line number and
+        # position of each element.
+        markup = "\n   <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>"
+        soup = self.soup(markup)
+        self.assertEqual(2, soup.p.sourceline)
+        self.assertEqual(3, soup.p.sourcepos)
+        self.assertEqual("sourceline", soup.p.find('sourceline').name)
+
+        # You can deactivate this behavior.
+        soup = self.soup(markup, store_line_numbers=False)
+        self.assertEqual("sourceline", soup.p.sourceline.name)
+        self.assertEqual("sourcepos", soup.p.sourcepos.name)
+
+        
+class TestHTMLParserSubclass(SoupTest):
+    def test_error(self):
+        """Verify that our HTMLParser subclass implements error() in a way
+        that doesn't cause a crash.
+        """
+        parser = BeautifulSoupHTMLParser()
+        parser.error("don't crash")

lib/bs4/tests/test_lxml.py

@@ -0,0 +1,115 @@
+"""Tests to ensure that the lxml tree builder generates good trees."""
+
+import re
+import warnings
+
+try:
+    import lxml.etree
+    LXML_PRESENT = True
+    LXML_VERSION = lxml.etree.LXML_VERSION
+except ImportError as e:
+    LXML_PRESENT = False
+    LXML_VERSION = (0,)
+
+if LXML_PRESENT:
+    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
+
+from bs4 import (
+    BeautifulSoup,
+    BeautifulStoneSoup,
+    )
+from bs4.element import Comment, Doctype, SoupStrainer
+from bs4.testing import skipIf
+from bs4.tests import test_htmlparser
+from bs4.testing import (
+    HTMLTreeBuilderSmokeTest,
+    XMLTreeBuilderSmokeTest,
+    SoupTest,
+    skipIf,
+)
+
+@skipIf(
+    not LXML_PRESENT,
+    "lxml seems not to be present, not testing its tree builder.")
+class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
+    """See ``HTMLTreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return LXMLTreeBuilder
+
+    def test_out_of_range_entity(self):
+        self.assertSoupEquals(
+            "<p>foo&#10000000000000;bar</p>", "<p>foobar</p>")
+        self.assertSoupEquals(
+            "<p>foo&#x10000000000000;bar</p>", "<p>foobar</p>")
+        self.assertSoupEquals(
+            "<p>foo&#1000000000;bar</p>", "<p>foobar</p>")
+
+    def test_entities_in_foreign_document_encoding(self):
+        # We can't implement this case correctly because by the time we
+        # hear about markup like "&#147;", it's been (incorrectly) converted into
+        # a string like u'\x93'
+        pass
+        
+    # In lxml < 2.3.5, an empty doctype causes a segfault. Skip this
+    # test if an old version of lxml is installed.
+
+    @skipIf(
+        not LXML_PRESENT or LXML_VERSION < (2,3,5,0),
+        "Skipping doctype test for old version of lxml to avoid segfault.")
+    def test_empty_doctype(self):
+        soup = self.soup("<!DOCTYPE>")
+        doctype = soup.contents[0]
+        self.assertEqual("", doctype.strip())
+
+    def test_beautifulstonesoup_is_xml_parser(self):
+        # Make sure that the deprecated BSS class uses an xml builder
+        # if one is installed.
+        with warnings.catch_warnings(record=True) as w:
+            soup = BeautifulStoneSoup("<b />")
+        self.assertEqual("<b/>", str(soup.b))
+        self.assertTrue("BeautifulStoneSoup class is deprecated" in str(w[0].message))
+
+    def test_tracking_line_numbers(self):
+        # The lxml TreeBuilder cannot keep track of line numbers from
+        # the original markup. Even if you ask for line numbers, we
+        # don't have 'em.
+        #
+        # This means that if you have a tag like <sourceline> or
+        # <sourcepos>, attribute access will find it rather than
+        # giving you a numeric answer.
+        soup = self.soup(
+            "\n   <p>\n\n<sourceline>\n<b>text</b></sourceline><sourcepos></p>",
+            store_line_numbers=True
+        )
+        self.assertEqual("sourceline", soup.p.sourceline.name)
+        self.assertEqual("sourcepos", soup.p.sourcepos.name)
+        
+@skipIf(
+    not LXML_PRESENT,
+    "lxml seems not to be present, not testing its XML tree builder.")
+class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest):
+    """See ``HTMLTreeBuilderSmokeTest``."""
+
+    @property
+    def default_builder(self):
+        return LXMLTreeBuilderForXML
+
+    def test_namespace_indexing(self):
+        # We should not track un-prefixed namespaces as we can only hold one
+        # and it will be recognized as the default namespace by soupsieve,
+        # which may be confusing in some situations. When no namespace is provided
+        # for a selector, the default namespace (if defined) is assumed.
+
+        soup = self.soup(
+            '<?xml version="1.1"?>\n'
+            '<root>'
+            '<tag xmlns="http://unprefixed-namespace.com">content</tag>'
+            '<prefix:tag xmlns:prefix="http://prefixed-namespace.com">content</tag>'
+            '</root>'
+        )
+        self.assertEqual(
+            soup._namespaces,
+            {'xml': 'http://www.w3.org/XML/1998/namespace', 'prefix': 'http://prefixed-namespace.com'}
+        )

lib/bs4/tests/test_soup.py

@@ -0,0 +1,682 @@
+# -*- coding: utf-8 -*-
+"""Tests of Beautiful Soup as a whole."""
+
+from pdb import set_trace
+import logging
+import unittest
+import sys
+import tempfile
+
+from bs4 import (
+    BeautifulSoup,
+    BeautifulStoneSoup,
+)
+from bs4.builder import (
+    TreeBuilder,
+    ParserRejectedMarkup,
+)
+from bs4.element import (
+    CharsetMetaAttributeValue,
+    Comment,
+    ContentMetaAttributeValue,
+    SoupStrainer,
+    NamespacedAttribute,
+    Tag,
+    NavigableString,
+    )
+
+import bs4.dammit
+from bs4.dammit import (
+    EntitySubstitution,
+    UnicodeDammit,
+    EncodingDetector,
+)
+from bs4.testing import (
+    default_builder,
+    SoupTest,
+    skipIf,
+)
+import warnings
+
+try:
+    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
+    LXML_PRESENT = True
+except ImportError as e:
+    LXML_PRESENT = False
+
+PYTHON_3_PRE_3_2 = (sys.version_info[0] == 3 and sys.version_info < (3,2))
+
+class TestConstructor(SoupTest):
+
+    def test_short_unicode_input(self):
+        data = "<h1>éé</h1>"
+        soup = self.soup(data)
+        self.assertEqual("éé", soup.h1.string)
+
+    def test_embedded_null(self):
+        data = "<h1>foo\0bar</h1>"
+        soup = self.soup(data)
+        self.assertEqual("foo\0bar", soup.h1.string)
+
+    def test_exclude_encodings(self):
+        utf8_data = "Räksmörgås".encode("utf-8")
+        soup = self.soup(utf8_data, exclude_encodings=["utf-8"])
+        self.assertEqual("windows-1252", soup.original_encoding)
+
+    def test_custom_builder_class(self):
+        # Verify that you can pass in a custom Builder class and
+        # it'll be instantiated with the appropriate keyword arguments.
+        class Mock(object):
+            def __init__(self, **kwargs):
+                self.called_with = kwargs
+                self.is_xml = True
+                self.store_line_numbers = False
+                self.cdata_list_attributes = []
+                self.preserve_whitespace_tags = []
+            def initialize_soup(self, soup):
+                pass
+            def feed(self, markup):
+                self.fed = markup
+            def reset(self):
+                pass
+            def ignore(self, ignore):
+                pass
+            set_up_substitutions = can_be_empty_element = ignore
+            def prepare_markup(self, *args, **kwargs):
+                yield "prepared markup", "original encoding", "declared encoding", "contains replacement characters"
+                
+        kwargs = dict(
+            var="value",
+            # This is a deprecated BS3-era keyword argument, which
+            # will be stripped out.
+            convertEntities=True,
+        )
+        with warnings.catch_warnings(record=True):
+            soup = BeautifulSoup('', builder=Mock, **kwargs)
+        assert isinstance(soup.builder, Mock)
+        self.assertEqual(dict(var="value"), soup.builder.called_with)
+        self.assertEqual("prepared markup", soup.builder.fed)
+        
+        # You can also instantiate the TreeBuilder yourself. In this
+        # case, that specific object is used and any keyword arguments
+        # to the BeautifulSoup constructor are ignored.
+        builder = Mock(**kwargs)
+        with warnings.catch_warnings(record=True) as w:
+            soup = BeautifulSoup(
+                '', builder=builder, ignored_value=True,
+            )
+        msg = str(w[0].message)
+        assert msg.startswith("Keyword arguments to the BeautifulSoup constructor will be ignored.")
+        self.assertEqual(builder, soup.builder)
+        self.assertEqual(kwargs, builder.called_with)
+
+    def test_parser_markup_rejection(self):
+        # If markup is completely rejected by the parser, an
+        # explanatory ParserRejectedMarkup exception is raised.
+        class Mock(TreeBuilder):
+            def feed(self, *args, **kwargs):
+                raise ParserRejectedMarkup("Nope.")
+
+        def prepare_markup(self, *args, **kwargs):
+            # We're going to try two different ways of preparing this markup,
+            # but feed() will reject both of them.
+            yield markup, None, None, False
+            yield markup, None, None, False
+            
+        import re
+        self.assertRaisesRegex(
+            ParserRejectedMarkup,
+            "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.",
+            BeautifulSoup, '', builder=Mock,
+        )
+        
+    def test_cdata_list_attributes(self):
+        # Most attribute values are represented as scalars, but the
+        # HTML standard says that some attributes, like 'class' have
+        # space-separated lists as values.
+        markup = '<a id=" an id " class=" a class "></a>'
+        soup = self.soup(markup)
+
+        # Note that the spaces are stripped for 'class' but not for 'id'.
+        a = soup.a
+        self.assertEqual(" an id ", a['id'])
+        self.assertEqual(["a", "class"], a['class'])
+
+        # TreeBuilder takes an argument called 'mutli_valued_attributes'  which lets
+        # you customize or disable this. As always, you can customize the TreeBuilder
+        # by passing in a keyword argument to the BeautifulSoup constructor.
+        soup = self.soup(markup, builder=default_builder, multi_valued_attributes=None)
+        self.assertEqual(" a class ", soup.a['class'])
+
+        # Here are two ways of saying that `id` is a multi-valued
+        # attribute in this context, but 'class' is not.
+        for switcheroo in ({'*': 'id'}, {'a': 'id'}):
+            with warnings.catch_warnings(record=True) as w:
+                # This will create a warning about not explicitly
+                # specifying a parser, but we'll ignore it.
+                soup = self.soup(markup, builder=None, multi_valued_attributes=switcheroo)
+            a = soup.a
+            self.assertEqual(["an", "id"], a['id'])
+            self.assertEqual(" a class ", a['class'])
+
+    def test_replacement_classes(self):
+        # Test the ability to pass in replacements for element classes
+        # which will be used when building the tree.
+        class TagPlus(Tag):
+            pass
+
+        class StringPlus(NavigableString):
+            pass
+
+        class CommentPlus(Comment):
+            pass
+        
+        soup = self.soup(
+            "<a><b>foo</b>bar</a><!--whee-->",
+            element_classes = {
+                Tag: TagPlus,
+                NavigableString: StringPlus,
+                Comment: CommentPlus,
+            }
+        )
+
+        # The tree was built with TagPlus, StringPlus, and CommentPlus objects,
+        # rather than Tag, String, and Comment objects.
+        assert all(
+            isinstance(x, (TagPlus, StringPlus, CommentPlus))
+            for x in soup.recursiveChildGenerator()
+        )
+        
+class TestWarnings(SoupTest):
+
+    def _no_parser_specified(self, s, is_there=True):
+        v = s.startswith(BeautifulSoup.NO_PARSER_SPECIFIED_WARNING[:80])
+        self.assertTrue(v)
+
+    def test_warning_if_no_parser_specified(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup("<a><b></b></a>")
+        msg = str(w[0].message)
+        self._assert_no_parser_specified(msg)
+
+    def test_warning_if_parser_specified_too_vague(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup("<a><b></b></a>", "html")
+        msg = str(w[0].message)
+        self._assert_no_parser_specified(msg)
+
+    def test_no_warning_if_explicit_parser_specified(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup("<a><b></b></a>", "html.parser")
+        self.assertEqual([], w)
+
+    def test_parseOnlyThese_renamed_to_parse_only(self):
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup("<a><b></b></a>", parseOnlyThese=SoupStrainer("b"))
+        msg = str(w[0].message)
+        self.assertTrue("parseOnlyThese" in msg)
+        self.assertTrue("parse_only" in msg)
+        self.assertEqual(b"<b></b>", soup.encode())
+
+    def test_fromEncoding_renamed_to_from_encoding(self):
+        with warnings.catch_warnings(record=True) as w:
+            utf8 = b"\xc3\xa9"
+            soup = self.soup(utf8, fromEncoding="utf8")
+        msg = str(w[0].message)
+        self.assertTrue("fromEncoding" in msg)
+        self.assertTrue("from_encoding" in msg)
+        self.assertEqual("utf8", soup.original_encoding)
+
+    def test_unrecognized_keyword_argument(self):
+        self.assertRaises(
+            TypeError, self.soup, "<a>", no_such_argument=True)
+
+class TestWarnings(SoupTest):
+
+    def test_disk_file_warning(self):
+        filehandle = tempfile.NamedTemporaryFile()
+        filename = filehandle.name
+        try:
+            with warnings.catch_warnings(record=True) as w:
+                soup = self.soup(filename)
+            msg = str(w[0].message)
+            self.assertTrue("looks like a filename" in msg)
+        finally:
+            filehandle.close()
+
+        # The file no longer exists, so Beautiful Soup will no longer issue the warning.
+        with warnings.catch_warnings(record=True) as w:
+            soup = self.soup(filename)
+        self.assertEqual(0, len(w))
+
+    def test_url_warning_with_bytes_url(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            soup = self.soup(b"http://www.crummybytes.com/")
+        # Be aware this isn't the only warning that can be raised during
+        # execution..
+        self.assertTrue(any("looks like a URL" in str(w.message) 
+            for w in warning_list))
+
+    def test_url_warning_with_unicode_url(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            # note - this url must differ from the bytes one otherwise
+            # python's warnings system swallows the second warning
+            soup = self.soup("http://www.crummyunicode.com/")
+        self.assertTrue(any("looks like a URL" in str(w.message) 
+            for w in warning_list))
+
+    def test_url_warning_with_bytes_and_space(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            soup = self.soup(b"http://www.crummybytes.com/ is great")
+        self.assertFalse(any("looks like a URL" in str(w.message) 
+            for w in warning_list))
+
+    def test_url_warning_with_unicode_and_space(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            soup = self.soup("http://www.crummyuncode.com/ is great")
+        self.assertFalse(any("looks like a URL" in str(w.message) 
+            for w in warning_list))
+
+
+class TestSelectiveParsing(SoupTest):
+
+    def test_parse_with_soupstrainer(self):
+        markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>"
+        strainer = SoupStrainer("b")
+        soup = self.soup(markup, parse_only=strainer)
+        self.assertEqual(soup.encode(), b"<b>Yes</b><b>Yes <c>Yes</c></b>")
+
+
+class TestEntitySubstitution(unittest.TestCase):
+    """Standalone tests of the EntitySubstitution class."""
+    def setUp(self):
+        self.sub = EntitySubstitution
+
+    def test_simple_html_substitution(self):
+        # Unicode characters corresponding to named HTML entites
+        # are substituted, and no others.
+        s = "foo\u2200\N{SNOWMAN}\u00f5bar"
+        self.assertEqual(self.sub.substitute_html(s),
+                          "foo&forall;\N{SNOWMAN}&otilde;bar")
+
+    def test_smart_quote_substitution(self):
+        # MS smart quotes are a common source of frustration, so we
+        # give them a special test.
+        quotes = b"\x91\x92foo\x93\x94"
+        dammit = UnicodeDammit(quotes)
+        self.assertEqual(self.sub.substitute_html(dammit.markup),
+                          "&lsquo;&rsquo;foo&ldquo;&rdquo;")
+
+    def test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false(self):
+        s = 'Welcome to "my bar"'
+        self.assertEqual(self.sub.substitute_xml(s, False), s)
+
+    def test_xml_attribute_quoting_normally_uses_double_quotes(self):
+        self.assertEqual(self.sub.substitute_xml("Welcome", True),
+                          '"Welcome"')
+        self.assertEqual(self.sub.substitute_xml("Bob's Bar", True),
+                          '"Bob\'s Bar"')
+
+    def test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes(self):
+        s = 'Welcome to "my bar"'
+        self.assertEqual(self.sub.substitute_xml(s, True),
+                          "'Welcome to \"my bar\"'")
+
+    def test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes(self):
+        s = 'Welcome to "Bob\'s Bar"'
+        self.assertEqual(
+            self.sub.substitute_xml(s, True),
+            '"Welcome to &quot;Bob\'s Bar&quot;"')
+
+    def test_xml_quotes_arent_escaped_when_value_is_not_being_quoted(self):
+        quoted = 'Welcome to "Bob\'s Bar"'
+        self.assertEqual(self.sub.substitute_xml(quoted), quoted)
+
+    def test_xml_quoting_handles_angle_brackets(self):
+        self.assertEqual(
+            self.sub.substitute_xml("foo<bar>"),
+            "foo&lt;bar&gt;")
+
+    def test_xml_quoting_handles_ampersands(self):
+        self.assertEqual(self.sub.substitute_xml("AT&T"), "AT&amp;T")
+
+    def test_xml_quoting_including_ampersands_when_they_are_part_of_an_entity(self):
+        self.assertEqual(
+            self.sub.substitute_xml("&Aacute;T&T"),
+            "&amp;Aacute;T&amp;T")
+
+    def test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity(self):
+        self.assertEqual(
+            self.sub.substitute_xml_containing_entities("&Aacute;T&T"),
+            "&Aacute;T&amp;T")
+       
+    def test_quotes_not_html_substituted(self):
+        """There's no need to do this except inside attribute values."""
+        text = 'Bob\'s "bar"'
+        self.assertEqual(self.sub.substitute_html(text), text)
+
+
+class TestEncodingConversion(SoupTest):
+    # Test Beautiful Soup's ability to decode and encode from various
+    # encodings.
+
+    def setUp(self):
+        super(TestEncodingConversion, self).setUp()
+        self.unicode_data = '<html><head><meta charset="utf-8"/></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>'
+        self.utf8_data = self.unicode_data.encode("utf-8")
+        # Just so you know what it looks like.
+        self.assertEqual(
+            self.utf8_data,
+            b'<html><head><meta charset="utf-8"/></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>')
+
+    def test_ascii_in_unicode_out(self):
+        # ASCII input is converted to Unicode. The original_encoding
+        # attribute is set to 'utf-8', a superset of ASCII.
+        chardet = bs4.dammit.chardet_dammit
+        logging.disable(logging.WARNING)
+        try:
+            def noop(str):
+                return None
+            # Disable chardet, which will realize that the ASCII is ASCII.
+            bs4.dammit.chardet_dammit = noop
+            ascii = b"<foo>a</foo>"
+            soup_from_ascii = self.soup(ascii)
+            unicode_output = soup_from_ascii.decode()
+            self.assertTrue(isinstance(unicode_output, str))
+            self.assertEqual(unicode_output, self.document_for(ascii.decode()))
+            self.assertEqual(soup_from_ascii.original_encoding.lower(), "utf-8")
+        finally:
+            logging.disable(logging.NOTSET)
+            bs4.dammit.chardet_dammit = chardet
+
+    def test_unicode_in_unicode_out(self):
+        # Unicode input is left alone. The original_encoding attribute
+        # is not set.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEqual(soup_from_unicode.decode(), self.unicode_data)
+        self.assertEqual(soup_from_unicode.foo.string, 'Sacr\xe9 bleu!')
+        self.assertEqual(soup_from_unicode.original_encoding, None)
+
+    def test_utf8_in_unicode_out(self):
+        # UTF-8 input is converted to Unicode. The original_encoding
+        # attribute is set.
+        soup_from_utf8 = self.soup(self.utf8_data)
+        self.assertEqual(soup_from_utf8.decode(), self.unicode_data)
+        self.assertEqual(soup_from_utf8.foo.string, 'Sacr\xe9 bleu!')
+
+    def test_utf8_out(self):
+        # The internal data structures can be encoded as UTF-8.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data)
+
+    @skipIf(
+        PYTHON_3_PRE_3_2,
+        "Bad HTMLParser detected; skipping test of non-ASCII characters in attribute name.")
+    def test_attribute_name_containing_unicode_characters(self):
+        markup = '<div><a \N{SNOWMAN}="snowman"></a></div>'
+        self.assertEqual(self.soup(markup).div.encode("utf8"), markup.encode("utf8"))
+
+class TestUnicodeDammit(unittest.TestCase):
+    """Standalone tests of UnicodeDammit."""
+
+    def test_unicode_input(self):
+        markup = "I'm already Unicode! \N{SNOWMAN}"
+        dammit = UnicodeDammit(markup)
+        self.assertEqual(dammit.unicode_markup, markup)
+
+    def test_smart_quotes_to_unicode(self):
+        markup = b"<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup)
+        self.assertEqual(
+            dammit.unicode_markup, "<foo>\u2018\u2019\u201c\u201d</foo>")
+
+    def test_smart_quotes_to_xml_entities(self):
+        markup = b"<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup, smart_quotes_to="xml")
+        self.assertEqual(
+            dammit.unicode_markup, "<foo>&#x2018;&#x2019;&#x201C;&#x201D;</foo>")
+
+    def test_smart_quotes_to_html_entities(self):
+        markup = b"<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup, smart_quotes_to="html")
+        self.assertEqual(
+            dammit.unicode_markup, "<foo>&lsquo;&rsquo;&ldquo;&rdquo;</foo>")
+
+    def test_smart_quotes_to_ascii(self):
+        markup = b"<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup, smart_quotes_to="ascii")
+        self.assertEqual(
+            dammit.unicode_markup, """<foo>''""</foo>""")
+
+    def test_detect_utf8(self):
+        utf8 = b"Sacr\xc3\xa9 bleu! \xe2\x98\x83"
+        dammit = UnicodeDammit(utf8)
+        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+        self.assertEqual(dammit.unicode_markup, 'Sacr\xe9 bleu! \N{SNOWMAN}')
+
+
+    def test_convert_hebrew(self):
+        hebrew = b"\xed\xe5\xec\xf9"
+        dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
+        self.assertEqual(dammit.original_encoding.lower(), 'iso-8859-8')
+        self.assertEqual(dammit.unicode_markup, '\u05dd\u05d5\u05dc\u05e9')
+
+    def test_dont_see_smart_quotes_where_there_are_none(self):
+        utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
+        dammit = UnicodeDammit(utf_8)
+        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+        self.assertEqual(dammit.unicode_markup.encode("utf-8"), utf_8)
+
+    def test_ignore_inappropriate_codecs(self):
+        utf8_data = "Räksmörgås".encode("utf-8")
+        dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
+        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+
+    def test_ignore_invalid_codecs(self):
+        utf8_data = "Räksmörgås".encode("utf-8")
+        for bad_encoding in ['.utf8', '...', 'utF---16.!']:
+            dammit = UnicodeDammit(utf8_data, [bad_encoding])
+            self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
+
+    def test_exclude_encodings(self):
+        # This is UTF-8.
+        utf8_data = "Räksmörgås".encode("utf-8")
+
+        # But if we exclude UTF-8 from consideration, the guess is
+        # Windows-1252.
+        dammit = UnicodeDammit(utf8_data, exclude_encodings=["utf-8"])
+        self.assertEqual(dammit.original_encoding.lower(), 'windows-1252')
+
+        # And if we exclude that, there is no valid guess at all.
+        dammit = UnicodeDammit(
+            utf8_data, exclude_encodings=["utf-8", "windows-1252"])
+        self.assertEqual(dammit.original_encoding, None)
+
+    def test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_character(self):
+        detected = EncodingDetector(
+            b'<?xml version="1.0" encoding="UTF-\xdb" ?>')
+        encodings = list(detected.encodings)
+        assert 'utf-\N{REPLACEMENT CHARACTER}' in encodings
+
+    def test_detect_html5_style_meta_tag(self):
+
+        for data in (
+            b'<html><meta charset="euc-jp" /></html>',
+            b"<html><meta charset='euc-jp' /></html>",
+            b"<html><meta charset=euc-jp /></html>",
+            b"<html><meta charset=euc-jp/></html>"):
+            dammit = UnicodeDammit(data, is_html=True)
+            self.assertEqual(
+                "euc-jp", dammit.original_encoding)
+
+    def test_last_ditch_entity_replacement(self):
+        # This is a UTF-8 document that contains bytestrings
+        # completely incompatible with UTF-8 (ie. encoded with some other
+        # encoding).
+        #
+        # Since there is no consistent encoding for the document,
+        # Unicode, Dammit will eventually encode the document as UTF-8
+        # and encode the incompatible characters as REPLACEMENT
+        # CHARACTER.
+        #
+        # If chardet is installed, it will detect that the document
+        # can be converted into ISO-8859-1 without errors. This happens
+        # to be the wrong encoding, but it is a consistent encoding, so the
+        # code we're testing here won't run.
+        #
+        # So we temporarily disable chardet if it's present.
+        doc = b"""\357\273\277<?xml version="1.0" encoding="UTF-8"?>
+<html><b>\330\250\330\252\330\261</b>
+<i>\310\322\321\220\312\321\355\344</i></html>"""
+        chardet = bs4.dammit.chardet_dammit
+        logging.disable(logging.WARNING)
+        try:
+            def noop(str):
+                return None
+            bs4.dammit.chardet_dammit = noop
+            dammit = UnicodeDammit(doc)
+            self.assertEqual(True, dammit.contains_replacement_characters)
+            self.assertTrue("\ufffd" in dammit.unicode_markup)
+
+            soup = BeautifulSoup(doc, "html.parser")
+            self.assertTrue(soup.contains_replacement_characters)
+        finally:
+            logging.disable(logging.NOTSET)
+            bs4.dammit.chardet_dammit = chardet
+
+    def test_byte_order_mark_removed(self):
+        # A document written in UTF-16LE will have its byte order marker stripped.
+        data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
+        dammit = UnicodeDammit(data)
+        self.assertEqual("<a>áé</a>", dammit.unicode_markup)
+        self.assertEqual("utf-16le", dammit.original_encoding)
+
+    def test_detwingle(self):
+        # Here's a UTF8 document.
+        utf8 = ("\N{SNOWMAN}" * 3).encode("utf8")
+
+        # Here's a Windows-1252 document.
+        windows_1252 = (
+            "\N{LEFT DOUBLE QUOTATION MARK}Hi, I like Windows!"
+            "\N{RIGHT DOUBLE QUOTATION MARK}").encode("windows_1252")
+
+        # Through some unholy alchemy, they've been stuck together.
+        doc = utf8 + windows_1252 + utf8
+
+        # The document can't be turned into UTF-8:
+        self.assertRaises(UnicodeDecodeError, doc.decode, "utf8")
+
+        # Unicode, Dammit thinks the whole document is Windows-1252,
+        # and decodes it into "☃☃☃“Hi, I like Windows!”☃☃☃"
+
+        # But if we run it through fix_embedded_windows_1252, it's fixed:
+
+        fixed = UnicodeDammit.detwingle(doc)
+        self.assertEqual(
+            "☃☃☃“Hi, I like Windows!”☃☃☃", fixed.decode("utf8"))
+
+    def test_detwingle_ignores_multibyte_characters(self):
+        # Each of these characters has a UTF-8 representation ending
+        # in \x93. \x93 is a smart quote if interpreted as
+        # Windows-1252. But our code knows to skip over multibyte
+        # UTF-8 characters, so they'll survive the process unscathed.
+        for tricky_unicode_char in (
+            "\N{LATIN SMALL LIGATURE OE}", # 2-byte char '\xc5\x93'
+            "\N{LATIN SUBSCRIPT SMALL LETTER X}", # 3-byte char '\xe2\x82\x93'
+            "\xf0\x90\x90\x93", # This is a CJK character, not sure which one.
+            ):
+            input = tricky_unicode_char.encode("utf8")
+            self.assertTrue(input.endswith(b'\x93'))
+            output = UnicodeDammit.detwingle(input)
+            self.assertEqual(output, input)
+
+    def test_find_declared_encoding(self):
+        # Test our ability to find a declared encoding inside an
+        # XML or HTML document.
+        #
+        # Even if the document comes in as Unicode, it may be
+        # interesting to know what encoding was claimed
+        # originally.
+
+        html_unicode = '<html><head><meta charset="utf-8"></head></html>'
+        html_bytes = html_unicode.encode("ascii")
+
+        xml_unicode= '<?xml version="1.0" encoding="ISO-8859-1" ?>'
+        xml_bytes = xml_unicode.encode("ascii")
+
+        m = EncodingDetector.find_declared_encoding
+        self.assertEqual(None, m(html_unicode, is_html=False))
+        self.assertEqual("utf-8", m(html_unicode, is_html=True))
+        self.assertEqual("utf-8", m(html_bytes, is_html=True))
+
+        self.assertEqual("iso-8859-1", m(xml_unicode))
+        self.assertEqual("iso-8859-1", m(xml_bytes))
+
+        # Normally, only the first few kilobytes of a document are checked for
+        # an encoding.
+        spacer = b' ' * 5000
+        self.assertEqual(None, m(spacer + html_bytes))
+        self.assertEqual(None, m(spacer + xml_bytes))
+
+        # But you can tell find_declared_encoding to search an entire
+        # HTML document.
+        self.assertEqual(
+            "utf-8",
+            m(spacer + html_bytes, is_html=True, search_entire_document=True)
+        )
+
+        # The XML encoding declaration has to be the very first thing
+        # in the document. We'll allow whitespace before the document
+        # starts, but nothing else.
+        self.assertEqual(
+            "iso-8859-1",
+            m(xml_bytes, search_entire_document=True)
+        )
+        self.assertEqual(
+            None, m(b'a' + xml_bytes, search_entire_document=True)
+        )
+            
+class TestNamedspacedAttribute(SoupTest):
+
+    def test_name_may_be_none_or_missing(self):
+        a = NamespacedAttribute("xmlns", None)
+        self.assertEqual(a, "xmlns")
+
+        a = NamespacedAttribute("xmlns")
+        self.assertEqual(a, "xmlns")
+        
+    def test_attribute_is_equivalent_to_colon_separated_string(self):
+        a = NamespacedAttribute("a", "b")
+        self.assertEqual("a:b", a)
+
+    def test_attributes_are_equivalent_if_prefix_and_name_identical(self):
+        a = NamespacedAttribute("a", "b", "c")
+        b = NamespacedAttribute("a", "b", "c")
+        self.assertEqual(a, b)
+
+        # The actual namespace is not considered.
+        c = NamespacedAttribute("a", "b", None)
+        self.assertEqual(a, c)
+
+        # But name and prefix are important.
+        d = NamespacedAttribute("a", "z", "c")
+        self.assertNotEqual(a, d)
+
+        e = NamespacedAttribute("z", "b", "c")
+        self.assertNotEqual(a, e)
+
+
+class TestAttributeValueWithCharsetSubstitution(unittest.TestCase):
+
+    def test_content_meta_attribute_value(self):
+        value = CharsetMetaAttributeValue("euc-jp")
+        self.assertEqual("euc-jp", value)
+        self.assertEqual("euc-jp", value.original_value)
+        self.assertEqual("utf8", value.encode("utf8"))
+
+
+    def test_content_meta_attribute_value(self):
+        value = ContentMetaAttributeValue("text/html; charset=euc-jp")
+        self.assertEqual("text/html; charset=euc-jp", value)
+        self.assertEqual("text/html; charset=euc-jp", value.original_value)
+        self.assertEqual("text/html; charset=utf8", value.encode("utf8"))

lib/certgen.py

@@ -76,7 +76,7 @@ def createCertificate(req, issuerCertKey, serial, validityPeriod, digest="sha256
     cert.sign(issuerKey, digest)
     return cert
 
-def createSelfSignedCertificate((issuerName, issuerKey), serial, (notBefore, notAfter), altNames, digest="sha256"):
+def createSelfSignedCertificate(issuerName, issuerKey, serial, notBefore, notAfter, altNames, digest="sha256"):
     """
     Generate a certificate given a certificate request.
     Arguments: issuerName - The name of the issuer
@@ -100,7 +100,7 @@ def createSelfSignedCertificate((issuerName, issuerKey), serial, (notBefore, not
     cert.set_pubkey(issuerKey)
 
     if altNames:
-        cert.add_extensions([crypto.X509Extension("subjectAltName", False, altNames)])
+        cert.add_extensions([crypto.X509Extension(b"subjectAltName", False, altNames)])
 
     cert.sign(issuerKey, digest)
     return cert

lib/cheroot/__init__.py

@@ -0,0 +1,15 @@
+"""High-performance, pure-Python HTTP server used by CherryPy."""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+try:
+    import pkg_resources
+except ImportError:
+    pass
+
+
+try:
+    __version__ = pkg_resources.get_distribution('cheroot').version
+except Exception:
+    __version__ = 'unknown'

lib/cheroot/__main__.py

@@ -0,0 +1,6 @@
+"""Stub for accessing the Cheroot CLI tool."""
+
+from .cli import main
+
+if __name__ == '__main__':
+    main()

lib/cheroot/_compat.py

@@ -0,0 +1,110 @@
+"""Compatibility code for using Cheroot with various versions of Python."""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import platform
+import re
+
+import six
+
+try:
+    import ssl
+    IS_ABOVE_OPENSSL10 = ssl.OPENSSL_VERSION_INFO >= (1, 1)
+    del ssl
+except ImportError:
+    IS_ABOVE_OPENSSL10 = None
+
+
+IS_PYPY = platform.python_implementation() == 'PyPy'
+
+
+SYS_PLATFORM = platform.system()
+IS_WINDOWS = SYS_PLATFORM == 'Windows'
+IS_LINUX = SYS_PLATFORM == 'Linux'
+IS_MACOS = SYS_PLATFORM == 'Darwin'
+
+PLATFORM_ARCH = platform.machine()
+IS_PPC = PLATFORM_ARCH.startswith('ppc')
+
+
+if not six.PY2:
+    def ntob(n, encoding='ISO-8859-1'):
+        """Return the native string as bytes in the given encoding."""
+        assert_native(n)
+        # In Python 3, the native string type is unicode
+        return n.encode(encoding)
+
+    def ntou(n, encoding='ISO-8859-1'):
+        """Return the native string as unicode with the given encoding."""
+        assert_native(n)
+        # In Python 3, the native string type is unicode
+        return n
+
+    def bton(b, encoding='ISO-8859-1'):
+        """Return the byte string as native string in the given encoding."""
+        return b.decode(encoding)
+else:
+    # Python 2
+    def ntob(n, encoding='ISO-8859-1'):
+        """Return the native string as bytes in the given encoding."""
+        assert_native(n)
+        # In Python 2, the native string type is bytes. Assume it's already
+        # in the given encoding, which for ISO-8859-1 is almost always what
+        # was intended.
+        return n
+
+    def ntou(n, encoding='ISO-8859-1'):
+        """Return the native string as unicode with the given encoding."""
+        assert_native(n)
+        # In Python 2, the native string type is bytes.
+        # First, check for the special encoding 'escape'. The test suite uses
+        # this to signal that it wants to pass a string with embedded \uXXXX
+        # escapes, but without having to prefix it with u'' for Python 2,
+        # but no prefix for Python 3.
+        if encoding == 'escape':
+            return re.sub(
+                r'\\u([0-9a-zA-Z]{4})',
+                lambda m: six.unichr(int(m.group(1), 16)),
+                n.decode('ISO-8859-1'),
+            )
+        # Assume it's already in the given encoding, which for ISO-8859-1
+        # is almost always what was intended.
+        return n.decode(encoding)
+
+    def bton(b, encoding='ISO-8859-1'):
+        """Return the byte string as native string in the given encoding."""
+        return b
+
+
+def assert_native(n):
+    """Check whether the input is of nativ ``str`` type.
+
+    Raises:
+        TypeError: in case of failed check
+
+    """
+    if not isinstance(n, str):
+        raise TypeError('n must be a native str (got %s)' % type(n).__name__)
+
+
+if not six.PY2:
+    """Python 3 has memoryview builtin."""
+    # Python 2.7 has it backported, but socket.write() does
+    # str(memoryview(b'0' * 100)) -> <memory at 0x7fb6913a5588>
+    # instead of accessing it correctly.
+    memoryview = memoryview
+else:
+    """Link memoryview to buffer under Python 2."""
+    memoryview = buffer  # noqa: F821
+
+
+def extract_bytes(mv):
+    """Retrieve bytes out of memoryview/buffer or bytes."""
+    if isinstance(mv, memoryview):
+        return bytes(mv) if six.PY2 else mv.tobytes()
+
+    if isinstance(mv, bytes):
+        return mv
+
+    raise ValueError

lib/cheroot/cli.py

@@ -0,0 +1,234 @@
+"""Command line tool for starting a Cheroot WSGI/HTTP server instance.
+
+Basic usage::
+
+    # Start a server on 127.0.0.1:8000 with the default settings
+    # for the WSGI app myapp/wsgi.py:application()
+    cheroot myapp.wsgi
+
+    # Start a server on 0.0.0.0:9000 with 8 threads
+    # for the WSGI app myapp/wsgi.py:main_app()
+    cheroot myapp.wsgi:main_app --bind 0.0.0.0:9000 --threads 8
+
+    # Start a server for the cheroot.server.Gateway subclass
+    # myapp/gateway.py:HTTPGateway
+    cheroot myapp.gateway:HTTPGateway
+
+    # Start a server on the UNIX socket /var/spool/myapp.sock
+    cheroot myapp.wsgi --bind /var/spool/myapp.sock
+
+    # Start a server on the abstract UNIX socket CherootServer
+    cheroot myapp.wsgi --bind @CherootServer
+"""
+
+import argparse
+from importlib import import_module
+import os
+import sys
+import contextlib
+
+import six
+
+from . import server
+from . import wsgi
+
+
+__metaclass__ = type
+
+
+class BindLocation:
+    """A class for storing the bind location for a Cheroot instance."""
+
+
+class TCPSocket(BindLocation):
+    """TCPSocket."""
+
+    def __init__(self, address, port):
+        """Initialize.
+
+        Args:
+            address (str): Host name or IP address
+            port (int): TCP port number
+        """
+        self.bind_addr = address, port
+
+
+class UnixSocket(BindLocation):
+    """UnixSocket."""
+
+    def __init__(self, path):
+        """Initialize."""
+        self.bind_addr = path
+
+
+class AbstractSocket(BindLocation):
+    """AbstractSocket."""
+
+    def __init__(self, addr):
+        """Initialize."""
+        self.bind_addr = '\0{}'.format(self.abstract_socket)
+
+
+class Application:
+    """Application."""
+
+    @classmethod
+    def resolve(cls, full_path):
+        """Read WSGI app/Gateway path string and import application module."""
+        mod_path, _, app_path = full_path.partition(':')
+        app = getattr(import_module(mod_path), app_path or 'application')
+
+        with contextlib.suppress(TypeError):
+            if issubclass(app, server.Gateway):
+                return GatewayYo(app)
+
+        return cls(app)
+
+    def __init__(self, wsgi_app):
+        """Initialize."""
+        if not callable(wsgi_app):
+            raise TypeError(
+                'Application must be a callable object or '
+                'cheroot.server.Gateway subclass',
+            )
+        self.wsgi_app = wsgi_app
+
+    def server_args(self, parsed_args):
+        """Return keyword args for Server class."""
+        args = {
+            arg: value
+            for arg, value in vars(parsed_args).items()
+            if not arg.startswith('_') and value is not None
+        }
+        args.update(vars(self))
+        return args
+
+    def server(self, parsed_args):
+        """Server."""
+        return wsgi.Server(**self.server_args(parsed_args))
+
+
+class GatewayYo:
+    """Gateway."""
+
+    def __init__(self, gateway):
+        """Init."""
+        self.gateway = gateway
+
+    def server(self, parsed_args):
+        """Server."""
+        server_args = vars(self)
+        server_args['bind_addr'] = parsed_args['bind_addr']
+        if parsed_args.max is not None:
+            server_args['maxthreads'] = parsed_args.max
+        if parsed_args.numthreads is not None:
+            server_args['minthreads'] = parsed_args.numthreads
+        return server.HTTPServer(**server_args)
+
+
+def parse_wsgi_bind_location(bind_addr_string):
+    """Convert bind address string to a BindLocation."""
+    # try and match for an IP/hostname and port
+    match = six.moves.urllib.parse.urlparse('//{}'.format(bind_addr_string))
+    try:
+        addr = match.hostname
+        port = match.port
+        if addr is not None or port is not None:
+            return TCPSocket(addr, port)
+    except ValueError:
+        pass
+
+    # else, assume a UNIX socket path
+    # if the string begins with an @ symbol, use an abstract socket
+    if bind_addr_string.startswith('@'):
+        return AbstractSocket(bind_addr_string[1:])
+    return UnixSocket(path=bind_addr_string)
+
+
+def parse_wsgi_bind_addr(bind_addr_string):
+    """Convert bind address string to bind address parameter."""
+    return parse_wsgi_bind_location(bind_addr_string).bind_addr
+
+
+_arg_spec = {
+    '_wsgi_app': dict(
+        metavar='APP_MODULE',
+        type=Application.resolve,
+        help='WSGI application callable or cheroot.server.Gateway subclass',
+    ),
+    '--bind': dict(
+        metavar='ADDRESS',
+        dest='bind_addr',
+        type=parse_wsgi_bind_addr,
+        default='[::1]:8000',
+        help='Network interface to listen on (default: [::1]:8000)',
+    ),
+    '--chdir': dict(
+        metavar='PATH',
+        type=os.chdir,
+        help='Set the working directory',
+    ),
+    '--server-name': dict(
+        dest='server_name',
+        type=str,
+        help='Web server name to be advertised via Server HTTP header',
+    ),
+    '--threads': dict(
+        metavar='INT',
+        dest='numthreads',
+        type=int,
+        help='Minimum number of worker threads',
+    ),
+    '--max-threads': dict(
+        metavar='INT',
+        dest='max',
+        type=int,
+        help='Maximum number of worker threads',
+    ),
+    '--timeout': dict(
+        metavar='INT',
+        dest='timeout',
+        type=int,
+        help='Timeout in seconds for accepted connections',
+    ),
+    '--shutdown-timeout': dict(
+        metavar='INT',
+        dest='shutdown_timeout',
+        type=int,
+        help='Time in seconds to wait for worker threads to cleanly exit',
+    ),
+    '--request-queue-size': dict(
+        metavar='INT',
+        dest='request_queue_size',
+        type=int,
+        help='Maximum number of queued connections',
+    ),
+    '--accepted-queue-size': dict(
+        metavar='INT',
+        dest='accepted_queue_size',
+        type=int,
+        help='Maximum number of active requests in queue',
+    ),
+    '--accepted-queue-timeout': dict(
+        metavar='INT',
+        dest='accepted_queue_timeout',
+        type=int,
+        help='Timeout in seconds for putting requests into queue',
+    ),
+}
+
+
+def main():
+    """Create a new Cheroot instance with arguments from the command line."""
+    parser = argparse.ArgumentParser(
+        description='Start an instance of the Cheroot WSGI/HTTP server.',
+    )
+    for arg, spec in _arg_spec.items():
+        parser.add_argument(arg, **spec)
+    raw_args = parser.parse_args()
+
+    # ensure cwd in sys.path
+    '' in sys.path or sys.path.insert(0, '')
+
+    # create a server based on the arguments provided
+    raw_args._wsgi_app.server(raw_args).safe_start()

lib/cheroot/connections.py

@@ -0,0 +1,279 @@
+"""Utilities to manage open connections."""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import io
+import os
+import select
+import socket
+import time
+
+from . import errors
+from .makefile import MakeFile
+
+import six
+
+try:
+    import fcntl
+except ImportError:
+    try:
+        from ctypes import windll, WinError
+        import ctypes.wintypes
+        _SetHandleInformation = windll.kernel32.SetHandleInformation
+        _SetHandleInformation.argtypes = [
+            ctypes.wintypes.HANDLE,
+            ctypes.wintypes.DWORD,
+            ctypes.wintypes.DWORD,
+        ]
+        _SetHandleInformation.restype = ctypes.wintypes.BOOL
+    except ImportError:
+        def prevent_socket_inheritance(sock):
+            """Stub inheritance prevention.
+
+            Dummy function, since neither fcntl nor ctypes are available.
+            """
+            pass
+    else:
+        def prevent_socket_inheritance(sock):
+            """Mark the given socket fd as non-inheritable (Windows)."""
+            if not _SetHandleInformation(sock.fileno(), 1, 0):
+                raise WinError()
+else:
+    def prevent_socket_inheritance(sock):
+        """Mark the given socket fd as non-inheritable (POSIX)."""
+        fd = sock.fileno()
+        old_flags = fcntl.fcntl(fd, fcntl.F_GETFD)
+        fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC)
+
+
+class ConnectionManager:
+    """Class which manages HTTPConnection objects.
+
+    This is for connections which are being kept-alive for follow-up requests.
+    """
+
+    def __init__(self, server):
+        """Initialize ConnectionManager object.
+
+        Args:
+            server (cheroot.server.HTTPServer): web server object
+                that uses this ConnectionManager instance.
+        """
+        self.server = server
+        self.connections = []
+
+    def put(self, conn):
+        """Put idle connection into the ConnectionManager to be managed.
+
+        Args:
+            conn (cheroot.server.HTTPConnection): HTTP connection
+                to be managed.
+        """
+        conn.last_used = time.time()
+        conn.ready_with_data = conn.rfile.has_data()
+        self.connections.append(conn)
+
+    def expire(self):
+        """Expire least recently used connections.
+
+        This happens if there are either too many open connections, or if the
+        connections have been timed out.
+
+        This should be called periodically.
+        """
+        if not self.connections:
+            return
+
+        # Look at the first connection - if it can be closed, then do
+        # that, and wait for get_conn to return it.
+        conn = self.connections[0]
+        if conn.closeable:
+            return
+
+        # Too many connections?
+        ka_limit = self.server.keep_alive_conn_limit
+        if ka_limit is not None and len(self.connections) > ka_limit:
+            conn.closeable = True
+            return
+
+        # Connection too old?
+        if (conn.last_used + self.server.timeout) < time.time():
+            conn.closeable = True
+            return
+
+    def get_conn(self, server_socket):
+        """Return a HTTPConnection object which is ready to be handled.
+
+        A connection returned by this method should be ready for a worker
+        to handle it. If there are no connections ready, None will be
+        returned.
+
+        Any connection returned by this method will need to be `put`
+        back if it should be examined again for another request.
+
+        Args:
+            server_socket (socket.socket): Socket to listen to for new
+            connections.
+        Returns:
+            cheroot.server.HTTPConnection instance, or None.
+
+        """
+        # Grab file descriptors from sockets, but stop if we find a
+        # connection which is already marked as ready.
+        socket_dict = {}
+        for conn in self.connections:
+            if conn.closeable or conn.ready_with_data:
+                break
+            socket_dict[conn.socket.fileno()] = conn
+        else:
+            # No ready connection.
+            conn = None
+
+        # We have a connection ready for use.
+        if conn:
+            self.connections.remove(conn)
+            return conn
+
+        # Will require a select call.
+        ss_fileno = server_socket.fileno()
+        socket_dict[ss_fileno] = server_socket
+        try:
+            rlist, _, _ = select.select(list(socket_dict), [], [], 0.1)
+            # No available socket.
+            if not rlist:
+                return None
+        except OSError:
+            # Mark any connection which no longer appears valid.
+            for fno, conn in list(socket_dict.items()):
+                # If the server socket is invalid, we'll just ignore it and
+                # wait to be shutdown.
+                if fno == ss_fileno:
+                    continue
+                try:
+                    os.fstat(fno)
+                except OSError:
+                    # Socket is invalid, close the connection, insert at
+                    # the front.
+                    self.connections.remove(conn)
+                    self.connections.insert(0, conn)
+                    conn.closeable = True
+
+            # Wait for the next tick to occur.
+            return None
+
+        try:
+            # See if we have a new connection coming in.
+            rlist.remove(ss_fileno)
+        except ValueError:
+            # No new connection, but reuse existing socket.
+            conn = socket_dict[rlist.pop()]
+        else:
+            conn = server_socket
+
+        # All remaining connections in rlist should be marked as ready.
+        for fno in rlist:
+            socket_dict[fno].ready_with_data = True
+
+        # New connection.
+        if conn is server_socket:
+            return self._from_server_socket(server_socket)
+
+        self.connections.remove(conn)
+        return conn
+
+    def _from_server_socket(self, server_socket):
+        try:
+            s, addr = server_socket.accept()
+            if self.server.stats['Enabled']:
+                self.server.stats['Accepts'] += 1
+            prevent_socket_inheritance(s)
+            if hasattr(s, 'settimeout'):
+                s.settimeout(self.server.timeout)
+
+            mf = MakeFile
+            ssl_env = {}
+            # if ssl cert and key are set, we try to be a secure HTTP server
+            if self.server.ssl_adapter is not None:
+                try:
+                    s, ssl_env = self.server.ssl_adapter.wrap(s)
+                except errors.NoSSLError:
+                    msg = (
+                        'The client sent a plain HTTP request, but '
+                        'this server only speaks HTTPS on this port.'
+                    )
+                    buf = [
+                        '%s 400 Bad Request\r\n' % self.server.protocol,
+                        'Content-Length: %s\r\n' % len(msg),
+                        'Content-Type: text/plain\r\n\r\n',
+                        msg,
+                    ]
+
+                    sock_to_make = s if not six.PY2 else s._sock
+                    wfile = mf(sock_to_make, 'wb', io.DEFAULT_BUFFER_SIZE)
+                    try:
+                        wfile.write(''.join(buf).encode('ISO-8859-1'))
+                    except socket.error as ex:
+                        if ex.args[0] not in errors.socket_errors_to_ignore:
+                            raise
+                    return
+                if not s:
+                    return
+                mf = self.server.ssl_adapter.makefile
+                # Re-apply our timeout since we may have a new socket object
+                if hasattr(s, 'settimeout'):
+                    s.settimeout(self.server.timeout)
+
+            conn = self.server.ConnectionClass(self.server, s, mf)
+
+            if not isinstance(
+                    self.server.bind_addr,
+                    (six.text_type, six.binary_type),
+            ):
+                # optional values
+                # Until we do DNS lookups, omit REMOTE_HOST
+                if addr is None:  # sometimes this can happen
+                    # figure out if AF_INET or AF_INET6.
+                    if len(s.getsockname()) == 2:
+                        # AF_INET
+                        addr = ('0.0.0.0', 0)
+                    else:
+                        # AF_INET6
+                        addr = ('::', 0)
+                conn.remote_addr = addr[0]
+                conn.remote_port = addr[1]
+
+            conn.ssl_env = ssl_env
+            return conn
+
+        except socket.timeout:
+            # The only reason for the timeout in start() is so we can
+            # notice keyboard interrupts on Win32, which don't interrupt
+            # accept() by default
+            return
+        except socket.error as ex:
+            if self.server.stats['Enabled']:
+                self.server.stats['Socket Errors'] += 1
+            if ex.args[0] in errors.socket_error_eintr:
+                # I *think* this is right. EINTR should occur when a signal
+                # is received during the accept() call; all docs say retry
+                # the call, and I *think* I'm reading it right that Python
+                # will then go ahead and poll for and handle the signal
+                # elsewhere. See
+                # https://github.com/cherrypy/cherrypy/issues/707.
+                return
+            if ex.args[0] in errors.socket_errors_nonblocking:
+                # Just try again. See
+                # https://github.com/cherrypy/cherrypy/issues/479.
+                return
+            if ex.args[0] in errors.socket_errors_to_ignore:
+                # Our socket was closed.
+                # See https://github.com/cherrypy/cherrypy/issues/686.
+                return
+            raise
+
+    def close(self):
+        """Close all monitored connections."""
+        for conn in self.connections[:]:
+            conn.close()
+        self.connections = []

lib/cheroot/errors.py

@@ -0,0 +1,58 @@
+"""Collection of exceptions raised and/or processed by Cheroot."""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import errno
+import sys
+
+
+class MaxSizeExceeded(Exception):
+    """Exception raised when a client sends more data then acceptable within limit.
+
+    Depends on ``request.body.maxbytes`` config option if used within CherryPy
+    """
+
+
+class NoSSLError(Exception):
+    """Exception raised when a client speaks HTTP to an HTTPS socket."""
+
+
+class FatalSSLAlert(Exception):
+    """Exception raised when the SSL implementation signals a fatal alert."""
+
+
+def plat_specific_errors(*errnames):
+    """Return error numbers for all errors in errnames on this platform.
+
+    The 'errno' module contains different global constants depending on
+    the specific platform (OS). This function will return the list of
+    numeric values for a given list of potential names.
+    """
+    missing_attr = set([None, ])
+    unique_nums = set(getattr(errno, k, None) for k in errnames)
+    return list(unique_nums - missing_attr)
+
+
+socket_error_eintr = plat_specific_errors('EINTR', 'WSAEINTR')
+
+socket_errors_to_ignore = plat_specific_errors(
+    'EPIPE',
+    'EBADF', 'WSAEBADF',
+    'ENOTSOCK', 'WSAENOTSOCK',
+    'ETIMEDOUT', 'WSAETIMEDOUT',
+    'ECONNREFUSED', 'WSAECONNREFUSED',
+    'ECONNRESET', 'WSAECONNRESET',
+    'ECONNABORTED', 'WSAECONNABORTED',
+    'ENETRESET', 'WSAENETRESET',
+    'EHOSTDOWN', 'EHOSTUNREACH',
+)
+socket_errors_to_ignore.append('timed out')
+socket_errors_to_ignore.append('The read operation timed out')
+socket_errors_nonblocking = plat_specific_errors(
+    'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK',
+)
+
+if sys.platform == 'darwin':
+    socket_errors_to_ignore.extend(plat_specific_errors('EPROTOTYPE'))
+    socket_errors_nonblocking.extend(plat_specific_errors('EPROTOTYPE'))

lib/cheroot/makefile.py

@@ -0,0 +1,447 @@
+"""Socket file object."""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import socket
+
+try:
+    # prefer slower Python-based io module
+    import _pyio as io
+except ImportError:
+    # Python 2.6
+    import io
+
+import six
+
+from . import errors
+from ._compat import extract_bytes, memoryview
+
+
+# Write only 16K at a time to sockets
+SOCK_WRITE_BLOCKSIZE = 16384
+
+
+class BufferedWriter(io.BufferedWriter):
+    """Faux file object attached to a socket object."""
+
+    def write(self, b):
+        """Write bytes to buffer."""
+        self._checkClosed()
+        if isinstance(b, str):
+            raise TypeError("can't write str to binary stream")
+
+        with self._write_lock:
+            self._write_buf.extend(b)
+            self._flush_unlocked()
+            return len(b)
+
+    def _flush_unlocked(self):
+        self._checkClosed('flush of closed file')
+        while self._write_buf:
+            try:
+                # ssl sockets only except 'bytes', not bytearrays
+                # so perhaps we should conditionally wrap this for perf?
+                n = self.raw.write(bytes(self._write_buf))
+            except io.BlockingIOError as e:
+                n = e.characters_written
+            del self._write_buf[:n]
+
+
+class MakeFile_PY2(getattr(socket, '_fileobject', object)):
+    """Faux file object attached to a socket object."""
+
+    def __init__(self, *args, **kwargs):
+        """Initialize faux file object."""
+        self.bytes_read = 0
+        self.bytes_written = 0
+        socket._fileobject.__init__(self, *args, **kwargs)
+        self._refcount = 0
+
+    def _reuse(self):
+        self._refcount += 1
+
+    def _drop(self):
+        if self._refcount < 0:
+            self.close()
+        else:
+            self._refcount -= 1
+
+    def write(self, data):
+        """Sendall for non-blocking sockets."""
+        bytes_sent = 0
+        data_mv = memoryview(data)
+        payload_size = len(data_mv)
+        while bytes_sent < payload_size:
+            try:
+                bytes_sent += self.send(
+                    data_mv[bytes_sent:bytes_sent + SOCK_WRITE_BLOCKSIZE],
+                )
+            except socket.error as e:
+                if e.args[0] not in errors.socket_errors_nonblocking:
+                    raise
+
+    def send(self, data):
+        """Send some part of message to the socket."""
+        bytes_sent = self._sock.send(extract_bytes(data))
+        self.bytes_written += bytes_sent
+        return bytes_sent
+
+    def flush(self):
+        """Write all data from buffer to socket and reset write buffer."""
+        if self._wbuf:
+            buffer = ''.join(self._wbuf)
+            self._wbuf = []
+            self.write(buffer)
+
+    def recv(self, size):
+        """Receive message of a size from the socket."""
+        while True:
+            try:
+                data = self._sock.recv(size)
+                self.bytes_read += len(data)
+                return data
+            except socket.error as e:
+                what = (
+                    e.args[0] not in errors.socket_errors_nonblocking
+                    and e.args[0] not in errors.socket_error_eintr
+                )
+                if what:
+                    raise
+
+    class FauxSocket:
+        """Faux socket with the minimal interface required by pypy."""
+
+        def _reuse(self):
+            pass
+
+    _fileobject_uses_str_type = six.PY2 and isinstance(
+        socket._fileobject(FauxSocket())._rbuf, six.string_types,
+    )
+
+    # FauxSocket is no longer needed
+    del FauxSocket
+
+    if not _fileobject_uses_str_type:
+        def read(self, size=-1):
+            """Read data from the socket to buffer."""
+            # Use max, disallow tiny reads in a loop as they are very
+            # inefficient.
+            # We never leave read() with any leftover data from a new recv()
+            # call in our internal buffer.
+            rbufsize = max(self._rbufsize, self.default_bufsize)
+            # Our use of StringIO rather than lists of string objects returned
+            # by recv() minimizes memory usage and fragmentation that occurs
+            # when rbufsize is large compared to the typical return value of
+            # recv().
+            buf = self._rbuf
+            buf.seek(0, 2)  # seek end
+            if size < 0:
+                # Read until EOF
+                # reset _rbuf.  we consume it via buf.
+                self._rbuf = io.BytesIO()
+                while True:
+                    data = self.recv(rbufsize)
+                    if not data:
+                        break
+                    buf.write(data)
+                return buf.getvalue()
+            else:
+                # Read until size bytes or EOF seen, whichever comes first
+                buf_len = buf.tell()
+                if buf_len >= size:
+                    # Already have size bytes in our buffer?  Extract and
+                    # return.
+                    buf.seek(0)
+                    rv = buf.read(size)
+                    self._rbuf = io.BytesIO()
+                    self._rbuf.write(buf.read())
+                    return rv
+
+                # reset _rbuf.  we consume it via buf.
+                self._rbuf = io.BytesIO()
+                while True:
+                    left = size - buf_len
+                    # recv() will malloc the amount of memory given as its
+                    # parameter even though it often returns much less data
+                    # than that.  The returned data string is short lived
+                    # as we copy it into a StringIO and free it.  This avoids
+                    # fragmentation issues on many platforms.
+                    data = self.recv(left)
+                    if not data:
+                        break
+                    n = len(data)
+                    if n == size and not buf_len:
+                        # Shortcut.  Avoid buffer data copies when:
+                        # - We have no data in our buffer.
+                        # AND
+                        # - Our call to recv returned exactly the
+                        #   number of bytes we were asked to read.
+                        return data
+                    if n == left:
+                        buf.write(data)
+                        del data  # explicit free
+                        break
+                    assert n <= left, 'recv(%d) returned %d bytes' % (left, n)
+                    buf.write(data)
+                    buf_len += n
+                    del data  # explicit free
+                    # assert buf_len == buf.tell()
+                return buf.getvalue()
+
+        def readline(self, size=-1):
+            """Read line from the socket to buffer."""
+            buf = self._rbuf
+            buf.seek(0, 2)  # seek end
+            if buf.tell() > 0:
+                # check if we already have it in our buffer
+                buf.seek(0)
+                bline = buf.readline(size)
+                if bline.endswith('\n') or len(bline) == size:
+                    self._rbuf = io.BytesIO()
+                    self._rbuf.write(buf.read())
+                    return bline
+                del bline
+            if size < 0:
+                # Read until \n or EOF, whichever comes first
+                if self._rbufsize <= 1:
+                    # Speed up unbuffered case
+                    buf.seek(0)
+                    buffers = [buf.read()]
+                    # reset _rbuf.  we consume it via buf.
+                    self._rbuf = io.BytesIO()
+                    data = None
+                    recv = self.recv
+                    while data != '\n':
+                        data = recv(1)
+                        if not data:
+                            break
+                        buffers.append(data)
+                    return ''.join(buffers)
+
+                buf.seek(0, 2)  # seek end
+                # reset _rbuf.  we consume it via buf.
+                self._rbuf = io.BytesIO()
+                while True:
+                    data = self.recv(self._rbufsize)
+                    if not data:
+                        break
+                    nl = data.find('\n')
+                    if nl >= 0:
+                        nl += 1
+                        buf.write(data[:nl])
+                        self._rbuf.write(data[nl:])
+                        del data
+                        break
+                    buf.write(data)
+                return buf.getvalue()
+
+            else:
+                # Read until size bytes or \n or EOF seen, whichever comes
+                # first
+                buf.seek(0, 2)  # seek end
+                buf_len = buf.tell()
+                if buf_len >= size:
+                    buf.seek(0)
+                    rv = buf.read(size)
+                    self._rbuf = io.BytesIO()
+                    self._rbuf.write(buf.read())
+                    return rv
+                # reset _rbuf.  we consume it via buf.
+                self._rbuf = io.BytesIO()
+                while True:
+                    data = self.recv(self._rbufsize)
+                    if not data:
+                        break
+                    left = size - buf_len
+                    # did we just receive a newline?
+                    nl = data.find('\n', 0, left)
+                    if nl >= 0:
+                        nl += 1
+                        # save the excess data to _rbuf
+                        self._rbuf.write(data[nl:])
+                        if buf_len:
+                            buf.write(data[:nl])
+                            break
+                        else:
+                            # Shortcut.  Avoid data copy through buf when
+                            # returning a substring of our first recv().
+                            return data[:nl]
+                    n = len(data)
+                    if n == size and not buf_len:
+                        # Shortcut.  Avoid data copy through buf when
+                        # returning exactly all of our first recv().
+                        return data
+                    if n >= left:
+                        buf.write(data[:left])
+                        self._rbuf.write(data[left:])
+                        break
+                    buf.write(data)
+                    buf_len += n
+                    # assert buf_len == buf.tell()
+                return buf.getvalue()
+
+        def has_data(self):
+            """Return true if there is buffered data to read."""
+            return bool(self._rbuf.getvalue())
+
+    else:
+        def read(self, size=-1):
+            """Read data from the socket to buffer."""
+            if size < 0:
+                # Read until EOF
+                buffers = [self._rbuf]
+                self._rbuf = ''
+                if self._rbufsize <= 1:
+                    recv_size = self.default_bufsize
+                else:
+                    recv_size = self._rbufsize
+
+                while True:
+                    data = self.recv(recv_size)
+                    if not data:
+                        break
+                    buffers.append(data)
+                return ''.join(buffers)
+            else:
+                # Read until size bytes or EOF seen, whichever comes first
+                data = self._rbuf
+                buf_len = len(data)
+                if buf_len >= size:
+                    self._rbuf = data[size:]
+                    return data[:size]
+                buffers = []
+                if data:
+                    buffers.append(data)
+                self._rbuf = ''
+                while True:
+                    left = size - buf_len
+                    recv_size = max(self._rbufsize, left)
+                    data = self.recv(recv_size)
+                    if not data:
+                        break
+                    buffers.append(data)
+                    n = len(data)
+                    if n >= left:
+                        self._rbuf = data[left:]
+                        buffers[-1] = data[:left]
+                        break
+                    buf_len += n
+                return ''.join(buffers)
+
+        def readline(self, size=-1):
+            """Read line from the socket to buffer."""
+            data = self._rbuf
+            if size < 0:
+                # Read until \n or EOF, whichever comes first
+                if self._rbufsize <= 1:
+                    # Speed up unbuffered case
+                    assert data == ''
+                    buffers = []
+                    while data != '\n':
+                        data = self.recv(1)
+                        if not data:
+                            break
+                        buffers.append(data)
+                    return ''.join(buffers)
+                nl = data.find('\n')
+                if nl >= 0:
+                    nl += 1
+                    self._rbuf = data[nl:]
+                    return data[:nl]
+                buffers = []
+                if data:
+                    buffers.append(data)
+                self._rbuf = ''
+                while True:
+                    data = self.recv(self._rbufsize)
+                    if not data:
+                        break
+                    buffers.append(data)
+                    nl = data.find('\n')
+                    if nl >= 0:
+                        nl += 1
+                        self._rbuf = data[nl:]
+                        buffers[-1] = data[:nl]
+                        break
+                return ''.join(buffers)
+            else:
+                # Read until size bytes or \n or EOF seen, whichever comes
+                # first
+                nl = data.find('\n', 0, size)
+                if nl >= 0:
+                    nl += 1
+                    self._rbuf = data[nl:]
+                    return data[:nl]
+                buf_len = len(data)
+                if buf_len >= size:
+                    self._rbuf = data[size:]
+                    return data[:size]
+                buffers = []
+                if data:
+                    buffers.append(data)
+                self._rbuf = ''
+                while True:
+                    data = self.recv(self._rbufsize)
+                    if not data:
+                        break
+                    buffers.append(data)
+                    left = size - buf_len
+                    nl = data.find('\n', 0, left)
+                    if nl >= 0:
+                        nl += 1
+                        self._rbuf = data[nl:]
+                        buffers[-1] = data[:nl]
+                        break
+                    n = len(data)
+                    if n >= left:
+                        self._rbuf = data[left:]
+                        buffers[-1] = data[:left]
+                        break
+                    buf_len += n
+                return ''.join(buffers)
+
+        def has_data(self):
+            """Return true if there is buffered data to read."""
+            return bool(self._rbuf)
+
+
+if not six.PY2:
+    class StreamReader(io.BufferedReader):
+        """Socket stream reader."""
+
+        def __init__(self, sock, mode='r', bufsize=io.DEFAULT_BUFFER_SIZE):
+            """Initialize socket stream reader."""
+            super().__init__(socket.SocketIO(sock, mode), bufsize)
+            self.bytes_read = 0
+
+        def read(self, *args, **kwargs):
+            """Capture bytes read."""
+            val = super().read(*args, **kwargs)
+            self.bytes_read += len(val)
+            return val
+
+        def has_data(self):
+            """Return true if there is buffered data to read."""
+            return len(self._read_buf) > self._read_pos
+
+    class StreamWriter(BufferedWriter):
+        """Socket stream writer."""
+
+        def __init__(self, sock, mode='w', bufsize=io.DEFAULT_BUFFER_SIZE):
+            """Initialize socket stream writer."""
+            super().__init__(socket.SocketIO(sock, mode), bufsize)
+            self.bytes_written = 0
+
+        def write(self, val, *args, **kwargs):
+            """Capture bytes written."""
+            res = super().write(val, *args, **kwargs)
+            self.bytes_written += len(val)
+            return res
+
+    def MakeFile(sock, mode='r', bufsize=io.DEFAULT_BUFFER_SIZE):
+        """File object attached to a socket object."""
+        cls = StreamReader if 'r' in mode else StreamWriter
+        return cls(sock, mode, bufsize)
+else:
+    StreamReader = StreamWriter = MakeFile = MakeFile_PY2

lib/cheroot/ssl/__init__.py

@@ -0,0 +1,52 @@
+"""Implementation of the SSL adapter base interface."""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from abc import ABCMeta, abstractmethod
+
+from six import add_metaclass
+
+
+@add_metaclass(ABCMeta)
+class Adapter:
+    """Base class for SSL driver library adapters.
+
+    Required methods:
+
+        * ``wrap(sock) -> (wrapped socket, ssl environ dict)``
+        * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) ->
+          socket file object``
+    """
+
+    @abstractmethod
+    def __init__(
+            self, certificate, private_key, certificate_chain=None,
+            ciphers=None,
+    ):
+        """Set up certificates, private key ciphers and reset context."""
+        self.certificate = certificate
+        self.private_key = private_key
+        self.certificate_chain = certificate_chain
+        self.ciphers = ciphers
+        self.context = None
+
+    @abstractmethod
+    def bind(self, sock):
+        """Wrap and return the given socket."""
+        return sock
+
+    @abstractmethod
+    def wrap(self, sock):
+        """Wrap and return the given socket, plus WSGI environ entries."""
+        raise NotImplementedError  # pragma: no cover
+
+    @abstractmethod
+    def get_environ(self):
+        """Return WSGI environ entries to be merged into each request."""
+        raise NotImplementedError  # pragma: no cover
+
+    @abstractmethod
+    def makefile(self, sock, mode='r', bufsize=-1):
+        """Return socket file object."""
+        raise NotImplementedError  # pragma: no cover

lib/cheroot/ssl/builtin.py

@@ -0,0 +1,210 @@
+"""
+A library for integrating Python's builtin ``ssl`` library with Cheroot.
+
+The ssl module must be importable for SSL functionality.
+
+To use this module, set ``HTTPServer.ssl_adapter`` to an instance of
+``BuiltinSSLAdapter``.
+"""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+try:
+    import ssl
+except ImportError:
+    ssl = None
+
+try:
+    from _pyio import DEFAULT_BUFFER_SIZE
+except ImportError:
+    try:
+        from io import DEFAULT_BUFFER_SIZE
+    except ImportError:
+        DEFAULT_BUFFER_SIZE = -1
+
+import six
+
+from . import Adapter
+from .. import errors
+from .._compat import IS_ABOVE_OPENSSL10
+from ..makefile import StreamReader, StreamWriter
+
+if six.PY2:
+    import socket
+    generic_socket_error = socket.error
+    del socket
+else:
+    generic_socket_error = OSError
+
+
+def _assert_ssl_exc_contains(exc, *msgs):
+    """Check whether SSL exception contains either of messages provided."""
+    if len(msgs) < 1:
+        raise TypeError(
+            '_assert_ssl_exc_contains() requires '
+            'at least one message to be passed.',
+        )
+    err_msg_lower = str(exc).lower()
+    return any(m.lower() in err_msg_lower for m in msgs)
+
+
+class BuiltinSSLAdapter(Adapter):
+    """A wrapper for integrating Python's builtin ssl module with Cheroot."""
+
+    certificate = None
+    """The filename of the server SSL certificate."""
+
+    private_key = None
+    """The filename of the server's private key file."""
+
+    certificate_chain = None
+    """The filename of the certificate chain file."""
+
+    context = None
+    """The ssl.SSLContext that will be used to wrap sockets."""
+
+    ciphers = None
+    """The ciphers list of SSL."""
+
+    CERT_KEY_TO_ENV = {
+        'subject': 'SSL_CLIENT_S_DN',
+        'issuer': 'SSL_CLIENT_I_DN',
+    }
+
+    CERT_KEY_TO_LDAP_CODE = {
+        'countryName': 'C',
+        'stateOrProvinceName': 'ST',
+        'localityName': 'L',
+        'organizationName': 'O',
+        'organizationalUnitName': 'OU',
+        'commonName': 'CN',
+        'emailAddress': 'Email',
+    }
+
+    def __init__(
+            self, certificate, private_key, certificate_chain=None,
+            ciphers=None,
+    ):
+        """Set up context in addition to base class properties if available."""
+        if ssl is None:
+            raise ImportError('You must install the ssl module to use HTTPS.')
+
+        super(BuiltinSSLAdapter, self).__init__(
+            certificate, private_key, certificate_chain, ciphers,
+        )
+
+        self.context = ssl.create_default_context(
+            purpose=ssl.Purpose.CLIENT_AUTH,
+            cafile=certificate_chain,
+        )
+        self.context.load_cert_chain(certificate, private_key)
+        if self.ciphers is not None:
+            self.context.set_ciphers(ciphers)
+
+    def bind(self, sock):
+        """Wrap and return the given socket."""
+        return super(BuiltinSSLAdapter, self).bind(sock)
+
+    def wrap(self, sock):
+        """Wrap and return the given socket, plus WSGI environ entries."""
+        EMPTY_RESULT = None, {}
+        try:
+            s = self.context.wrap_socket(
+                sock, do_handshake_on_connect=True, server_side=True,
+            )
+        except ssl.SSLError as ex:
+            if ex.errno == ssl.SSL_ERROR_EOF:
+                # This is almost certainly due to the cherrypy engine
+                # 'pinging' the socket to assert it's connectable;
+                # the 'ping' isn't SSL.
+                return EMPTY_RESULT
+            elif ex.errno == ssl.SSL_ERROR_SSL:
+                if _assert_ssl_exc_contains(ex, 'http request'):
+                    # The client is speaking HTTP to an HTTPS server.
+                    raise errors.NoSSLError
+
+                # Check if it's one of the known errors
+                # Errors that are caught by PyOpenSSL, but thrown by
+                # built-in ssl
+                _block_errors = (
+                    'unknown protocol', 'unknown ca', 'unknown_ca',
+                    'unknown error',
+                    'https proxy request', 'inappropriate fallback',
+                    'wrong version number',
+                    'no shared cipher', 'certificate unknown',
+                    'ccs received early',
+                    'certificate verify failed',  # client cert w/o trusted CA
+                )
+                if _assert_ssl_exc_contains(ex, *_block_errors):
+                    # Accepted error, let's pass
+                    return EMPTY_RESULT
+            elif _assert_ssl_exc_contains(ex, 'handshake operation timed out'):
+                # This error is thrown by builtin SSL after a timeout
+                # when client is speaking HTTP to an HTTPS server.
+                # The connection can safely be dropped.
+                return EMPTY_RESULT
+            raise
+        except generic_socket_error as exc:
+            """It is unclear why exactly this happens.
+
+            It's reproducible only with openssl>1.0 and stdlib ``ssl`` wrapper.
+            In CherryPy it's triggered by Checker plugin, which connects
+            to the app listening to the socket port in TLS mode via plain
+            HTTP during startup (from the same process).
+
+
+            Ref: https://github.com/cherrypy/cherrypy/issues/1618
+            """
+            is_error0 = exc.args == (0, 'Error')
+
+            if is_error0 and IS_ABOVE_OPENSSL10:
+                return EMPTY_RESULT
+            raise
+        return s, self.get_environ(s)
+
+    # TODO: fill this out more with mod ssl env
+    def get_environ(self, sock):
+        """Create WSGI environ entries to be merged into each request."""
+        cipher = sock.cipher()
+        ssl_environ = {
+            'wsgi.url_scheme': 'https',
+            'HTTPS': 'on',
+            'SSL_PROTOCOL': cipher[1],
+            'SSL_CIPHER': cipher[0],
+            # SSL_VERSION_INTERFACE     string  The mod_ssl program version
+            # SSL_VERSION_LIBRARY   string  The OpenSSL program version
+        }
+
+        if self.context and self.context.verify_mode != ssl.CERT_NONE:
+            client_cert = sock.getpeercert()
+            if client_cert:
+                for cert_key, env_var in self.CERT_KEY_TO_ENV.items():
+                    ssl_environ.update(
+                        self.env_dn_dict(env_var, client_cert.get(cert_key)),
+                    )
+
+        return ssl_environ
+
+    def env_dn_dict(self, env_prefix, cert_value):
+        """Return a dict of WSGI environment variables for a client cert DN.
+
+        E.g. SSL_CLIENT_S_DN_CN, SSL_CLIENT_S_DN_C, etc.
+        See SSL_CLIENT_S_DN_x509 at
+        https://httpd.apache.org/docs/2.4/mod/mod_ssl.html#envvars.
+        """
+        if not cert_value:
+            return {}
+
+        env = {}
+        for rdn in cert_value:
+            for attr_name, val in rdn:
+                attr_code = self.CERT_KEY_TO_LDAP_CODE.get(attr_name)
+                if attr_code:
+                    env['%s_%s' % (env_prefix, attr_code)] = val
+        return env
+
+    def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE):
+        """Return socket file object."""
+        cls = StreamReader if 'r' in mode else StreamWriter
+        return cls(sock, mode, bufsize)

lib/cheroot/ssl/pyopenssl.py

@@ -1,10 +1,11 @@
-"""A library for integrating pyOpenSSL with CherryPy.
+"""
+A library for integrating pyOpenSSL with Cheroot.
 
 The OpenSSL module must be importable for SSL functionality.
 You can obtain it from `here <https://launchpad.net/pyopenssl>`_.
 
-To use this module, set CherryPyWSGIServer.ssl_adapter to an instance of
-SSLAdapter. There are two ways to use SSL:
+To use this module, set HTTPServer.ssl_adapter to an instance of
+ssl.Adapter. There are two ways to use SSL:
 
 Method One
 ----------
@@ -30,26 +31,33 @@ and .certificate are both given and valid, they will be read, and the
 context will be automatically created from them.
 """
 
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
 import socket
 import threading
 import time
 
-from cherrypy import wsgiserver
+import six
 
 try:
     from OpenSSL import SSL
     from OpenSSL import crypto
-    if hasattr(SSL, 'Connection'):
-        SSLConnectionType = SSL.Connection
-    else:
-        SSLConnectionType = SSL.ConnectionType
+
+    try:
+        ssl_conn_type = SSL.Connection
+    except AttributeError:
+        ssl_conn_type = SSL.ConnectionType
 except ImportError:
     SSL = None
 
+from . import Adapter
+from .. import errors, server as cheroot_server
+from ..makefile import StreamReader, StreamWriter
 
-class SSL_fileobject(wsgiserver.CP_fileobject):
 
-    """SSL file object attached to a socket object."""
+class SSLFileobjectMixin:
+    """Base mixin for an SSL socket stream."""
 
     ssl_timeout = 3
     ssl_retry = .01
@@ -69,20 +77,21 @@ class SSL_fileobject(wsgiserver.CP_fileobject):
                 # the rest of the stack has no way of differentiating
                 # between a "new handshake" error and "client dropped".
                 # Note this isn't an endless loop: there's a timeout below.
+                # Ref: https://stackoverflow.com/a/5133568/595220
                 time.sleep(self.ssl_retry)
             except SSL.WantWriteError:
                 time.sleep(self.ssl_retry)
             except SSL.SysCallError as e:
                 if is_reader and e.args == (-1, 'Unexpected EOF'):
-                    return ""
+                    return b''
 
                 errnum = e.args[0]
-                if is_reader and errnum in wsgiserver.socket_errors_to_ignore:
-                    return ""
+                if is_reader and errnum in errors.socket_errors_to_ignore:
+                    return b''
                 raise socket.error(errnum)
             except SSL.Error as e:
                 if is_reader and e.args == (-1, 'Unexpected EOF'):
-                    return ""
+                    return b''
 
                 thirdarg = None
                 try:
@@ -92,69 +101,127 @@ class SSL_fileobject(wsgiserver.CP_fileobject):
 
                 if thirdarg == 'http request':
                     # The client is talking HTTP to an HTTPS server.
-                    raise wsgiserver.NoSSLError()
+                    raise errors.NoSSLError()
 
-                raise wsgiserver.FatalSSLAlert(*e.args)
-            except:
-                raise
+                raise errors.FatalSSLAlert(*e.args)
 
             if time.time() - start > self.ssl_timeout:
-                raise socket.timeout("timed out")
+                raise socket.timeout('timed out')
 
     def recv(self, size):
-        return self._safe_call(True, super(SSL_fileobject, self).recv, size)
+        """Receive message of a size from the socket."""
+        return self._safe_call(
+            True,
+            super(SSLFileobjectMixin, self).recv,
+            size,
+        )
+
+    def readline(self, size=-1):
+        """Receive message of a size from the socket.
+
+        Matches the following interface:
+        https://docs.python.org/3/library/io.html#io.IOBase.readline
+        """
+        return self._safe_call(
+            True,
+            super(SSLFileobjectMixin, self).readline,
+            size,
+        )
 
     def sendall(self, *args, **kwargs):
-        return self._safe_call(False, super(SSL_fileobject, self).sendall,
-                               *args, **kwargs)
+        """Send whole message to the socket."""
+        return self._safe_call(
+            False,
+            super(SSLFileobjectMixin, self).sendall,
+            *args, **kwargs
+        )
 
     def send(self, *args, **kwargs):
-        return self._safe_call(False, super(SSL_fileobject, self).send,
-                               *args, **kwargs)
+        """Send some part of message to the socket."""
+        return self._safe_call(
+            False,
+            super(SSLFileobjectMixin, self).send,
+            *args, **kwargs
+        )
 
 
+class SSLFileobjectStreamReader(SSLFileobjectMixin, StreamReader):
+    """SSL file object attached to a socket object."""
+
+
+class SSLFileobjectStreamWriter(SSLFileobjectMixin, StreamWriter):
+    """SSL file object attached to a socket object."""
+
+
+class SSLConnectionProxyMeta:
+    """Metaclass for generating a bunch of proxy methods."""
+
+    def __new__(mcl, name, bases, nmspc):
+        """Attach a list of proxy methods to a new class."""
+        proxy_methods = (
+            'get_context', 'pending', 'send', 'write', 'recv', 'read',
+            'renegotiate', 'bind', 'listen', 'connect', 'accept',
+            'setblocking', 'fileno', 'close', 'get_cipher_list',
+            'getpeername', 'getsockname', 'getsockopt', 'setsockopt',
+            'makefile', 'get_app_data', 'set_app_data', 'state_string',
+            'sock_shutdown', 'get_peer_certificate', 'want_read',
+            'want_write', 'set_connect_state', 'set_accept_state',
+            'connect_ex', 'sendall', 'settimeout', 'gettimeout',
+            'shutdown',
+        )
+        proxy_methods_no_args = (
+            'shutdown',
+        )
+
+        proxy_props = (
+            'family',
+        )
+
+        def lock_decorator(method):
+            """Create a proxy method for a new class."""
+            def proxy_wrapper(self, *args):
+                self._lock.acquire()
+                try:
+                    new_args = (
+                        args[:] if method not in proxy_methods_no_args else []
+                    )
+                    return getattr(self._ssl_conn, method)(*new_args)
+                finally:
+                    self._lock.release()
+            return proxy_wrapper
+        for m in proxy_methods:
+            nmspc[m] = lock_decorator(m)
+            nmspc[m].__name__ = m
+
+        def make_property(property_):
+            """Create a proxy method for a new class."""
+            def proxy_prop_wrapper(self):
+                return getattr(self._ssl_conn, property_)
+            proxy_prop_wrapper.__name__ = property_
+            return property(proxy_prop_wrapper)
+        for p in proxy_props:
+            nmspc[p] = make_property(p)
+
+        # Doesn't work via super() for some reason.
+        # Falling back to type() instead:
+        return type(name, bases, nmspc)
+
+
+@six.add_metaclass(SSLConnectionProxyMeta)
 class SSLConnection:
-
     """A thread-safe wrapper for an SSL.Connection.
 
     ``*args``: the arguments to create the wrapped ``SSL.Connection(*args)``.
     """
 
     def __init__(self, *args):
+        """Initialize SSLConnection instance."""
         self._ssl_conn = SSL.Connection(*args)
         self._lock = threading.RLock()
 
-    for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read',
-              'renegotiate', 'bind', 'listen', 'connect', 'accept',
-              'setblocking', 'fileno', 'close', 'get_cipher_list',
-              'getpeername', 'getsockname', 'getsockopt', 'setsockopt',
-              'makefile', 'get_app_data', 'set_app_data', 'state_string',
-              'sock_shutdown', 'get_peer_certificate', 'want_read',
-              'want_write', 'set_connect_state', 'set_accept_state',
-              'connect_ex', 'sendall', 'settimeout', 'gettimeout'):
-        exec("""def %s(self, *args):
-        self._lock.acquire()
-        try:
-            return self._ssl_conn.%s(*args)
-        finally:
-            self._lock.release()
-""" % (f, f))
 
-    def shutdown(self, *args):
-        self._lock.acquire()
-        try:
-            # pyOpenSSL.socket.shutdown takes no args
-            return self._ssl_conn.shutdown()
-        finally:
-            self._lock.release()
-
-
-class pyOpenSSLAdapter(wsgiserver.SSLAdapter):
-
-    """A wrapper for integrating pyOpenSSL with CherryPy."""
-
-    context = None
-    """An instance of SSL.Context."""
+class pyOpenSSLAdapter(Adapter):
+    """A wrapper for integrating pyOpenSSL with Cheroot."""
 
     certificate = None
     """The filename of the server SSL certificate."""
@@ -168,14 +235,24 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter):
     This is needed for cheaper "chained root" SSL certificates, and should be
     left as None if not required."""
 
-    def __init__(self, certificate, private_key, certificate_chain=None):
-        if SSL is None:
-            raise ImportError("You must install pyOpenSSL to use HTTPS.")
+    context = None
+    """An instance of SSL.Context."""
+
+    ciphers = None
+    """The ciphers list of SSL."""
+
+    def __init__(
+            self, certificate, private_key, certificate_chain=None,
+            ciphers=None,
+    ):
+        """Initialize OpenSSL Adapter instance."""
+        if SSL is None:
+            raise ImportError('You must install pyOpenSSL to use HTTPS.')
+
+        super(pyOpenSSLAdapter, self).__init__(
+            certificate, private_key, certificate_chain, ciphers,
+        )
 
-        self.context = None
-        self.certificate = certificate
-        self.private_key = private_key
-        self.certificate_chain = certificate_chain
         self._environ = None
 
     def bind(self, sock):
@@ -192,7 +269,7 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter):
 
     def get_context(self):
         """Return an SSL.Context from self attributes."""
-        # See http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/442473
+        # See https://code.activestate.com/recipes/442473/
         c = SSL.Context(SSL.SSLv23_METHOD)
         c.use_privatekey_file(self.private_key)
         if self.certificate_chain:
@@ -203,12 +280,12 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter):
     def get_environ(self):
         """Return WSGI environ entries to be merged into each request."""
         ssl_environ = {
-            "HTTPS": "on",
+            'HTTPS': 'on',
             # pyOpenSSL doesn't provide access to any of these AFAICT
             # 'SSL_PROTOCOL': 'SSLv2',
-            # SSL_CIPHER 	string 	The cipher specification name
-            # SSL_VERSION_INTERFACE 	string 	The mod_ssl program version
-            # SSL_VERSION_LIBRARY 	string 	The OpenSSL program version
+            # SSL_CIPHER    string  The cipher specification name
+            # SSL_VERSION_INTERFACE     string  The mod_ssl program version
+            # SSL_VERSION_LIBRARY   string  The OpenSSL program version
         }
 
         if self.certificate:
@@ -224,8 +301,10 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter):
                 #   Validity of server's certificate (end time),
             })
 
-            for prefix, dn in [("I", cert.get_issuer()),
-                               ("S", cert.get_subject())]:
+            for prefix, dn in [
+                ('I', cert.get_issuer()),
+                ('S', cert.get_subject()),
+            ]:
                 # X509Name objects don't seem to have a way to get the
                 # complete DN string. Use str() and slice it instead,
                 # because str(dn) == "<X509Name object '/C=US/ST=...'>"
@@ -237,9 +316,9 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter):
                 # The DN should be of the form: /k1=v1/k2=v2, but we must allow
                 # for any value to contain slashes itself (in a URL).
                 while dnstr:
-                    pos = dnstr.rfind("=")
+                    pos = dnstr.rfind('=')
                     dnstr, value = dnstr[:pos], dnstr[pos + 1:]
-                    pos = dnstr.rfind("/")
+                    pos = dnstr.rfind('/')
                     dnstr, key = dnstr[:pos], dnstr[pos + 1:]
                     if key and value:
                         wsgikey = 'SSL_SERVER_%s_DN_%s' % (prefix, key)
@@ -248,10 +327,17 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter):
         return ssl_environ
 
     def makefile(self, sock, mode='r', bufsize=-1):
-        if SSL and isinstance(sock, SSLConnectionType):
-            timeout = sock.gettimeout()
-            f = SSL_fileobject(sock, mode, bufsize)
-            f.ssl_timeout = timeout
-            return f
+        """Return socket file object."""
+        cls = (
+            SSLFileobjectStreamReader
+            if 'r' in mode else
+            SSLFileobjectStreamWriter
+        )
+        if SSL and isinstance(sock, ssl_conn_type):
+            wrapped_socket = cls(sock, mode, bufsize)
+            wrapped_socket.ssl_timeout = sock.gettimeout()
+            return wrapped_socket
+        # This is from past:
+        # TODO: figure out what it's meant for
         else:
-            return wsgiserver.CP_fileobject(sock, mode, bufsize)
+            return cheroot_server.CP_fileobject(sock, mode, bufsize)

lib/cheroot/test/__init__.py

@@ -0,0 +1 @@
+"""Cheroot test suite."""

lib/cheroot/test/conftest.py

@@ -0,0 +1,69 @@
+"""Pytest configuration module.
+
+Contains fixtures, which are tightly bound to the Cheroot framework
+itself, useless for end-users' app testing.
+"""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import threading
+import time
+
+import pytest
+
+from ..server import Gateway, HTTPServer
+from ..testing import (  # noqa: F401
+    native_server, wsgi_server,
+)
+from ..testing import get_server_client
+
+
+@pytest.fixture
+def wsgi_server_client(wsgi_server):  # noqa: F811
+    """Create a test client out of given WSGI server."""
+    return get_server_client(wsgi_server)
+
+
+@pytest.fixture
+def native_server_client(native_server):  # noqa: F811
+    """Create a test client out of given HTTP server."""
+    return get_server_client(native_server)
+
+
+@pytest.fixture
+def http_server():
+    """Provision a server creator as a fixture."""
+    def start_srv():
+        bind_addr = yield
+        if bind_addr is None:
+            return
+        httpserver = make_http_server(bind_addr)
+        yield httpserver
+        yield httpserver
+
+    srv_creator = iter(start_srv())
+    next(srv_creator)
+    yield srv_creator
+    try:
+        while True:
+            httpserver = next(srv_creator)
+            if httpserver is not None:
+                httpserver.stop()
+    except StopIteration:
+        pass
+
+
+def make_http_server(bind_addr):
+    """Create and start an HTTP server bound to bind_addr."""
+    httpserver = HTTPServer(
+        bind_addr=bind_addr,
+        gateway=Gateway,
+    )
+
+    threading.Thread(target=httpserver.safe_start).start()
+
+    while not httpserver.ready:
+        time.sleep(0.1)
+
+    return httpserver