v2.5.6

Added Silent Notification option for Telegram Agent

.dockerignore

@@ -3,6 +3,8 @@
 .gitignore
 contrib
 init-scripts
+package
 pylintrc
 *.md
 !CHANGELOG*.md
+start.bat

.github/workflows/publish-docker.yml

@@ -1,12 +1,15 @@
 name: Publish Docker
 on:
   push:
-    branches: [master, beta, nightly]
+    branches: [master, beta, nightly, python3]
     tags: [v*]
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
       - name: Prepare
         id: prepare
         run: |
@@ -25,20 +28,28 @@ jobs:
           echo ::set-output name=commit::${GITHUB_SHA}
           echo ::set-output name=build_date::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
           echo ::set-output name=docker_platforms::linux/amd64,linux/arm64,linux/arm
-          echo ::set-output name=docker_image::tautulli/tautulli
+          echo ::set-output name=docker_image::${{ secrets.DOCKER_REPO }}/tautulli
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: crazy-max/ghaction-docker-buildx@v1
+        uses: crazy-max/ghaction-docker-buildx@v3
         with:
-          version: latest
+          buildx-version: latest
 
-      - name: Checkout
+      - name: Cache Docker Layers
-        uses: actions/checkout@v2
+        id: cache
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
 
       - name: Docker Buildx (no push)
         run: |
           docker buildx build \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --cache-to "type=local,dest=/tmp/.buildx-cache" \
             --platform ${{ steps.prepare.outputs.docker_platforms }} \
             --output "type=image,push=false" \
             --build-arg "TAG=${{ steps.prepare.outputs.tag }}" \
@@ -59,6 +70,7 @@ jobs:
         if: success()
         run: |
           docker buildx build \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
             --platform ${{ steps.prepare.outputs.docker_platforms }} \
             --output "type=image,push=true" \
             --build-arg "TAG=${{ steps.prepare.outputs.tag }}" \
@@ -79,5 +91,5 @@ jobs:
         with:
           webhook: ${{ secrets.DISCORD_WEBHOOK }}
           status: ${{ job.status }}
-          job: ${{ github.workflow }}
+          title: ${{ github.workflow }}
           nofail: true

.github/workflows/publish-release.yml

@@ -1,28 +1,204 @@
 name: Publish Release
 on:
   push:
+    branches: [master, beta, nightly, python3]
     tags: [v*]
+
 jobs:
-  build:
+  build-windows:
-    runs-on: ubuntu-latest
+    runs-on: windows-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@master
+        uses: actions/checkout@v2
-      - name: Get Release Version
+
-        run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+      - name: Set Release Version
+        id: get_version
+        shell: bash
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION_NSIS=${GITHUB_REF#refs/tags/v}.1
+            echo ::set-output name=VERSION_NSIS::${VERSION_NSIS/%-beta.1/.0}
+            echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/v}
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+          else
+            echo ::set-output name=VERSION_NSIS::0.0.0.0
+            echo ::set-output name=VERSION::0.0.0
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_SHA::7}
+          fi
+          echo $GITHUB_SHA > version.txt
+
+      - name: Set Up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Cache Dependencies
+        id: cache_dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{ runner.os }}-pip-${{ hashFiles('package/requirements-windows.txt') }}
+          restore-keys: ${{ runner.os }}-pip-
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r package/requirements-windows.txt
+
+      - name: Build Package
+        run: |
+          pyinstaller -y ./package/Tautulli-windows.spec
+
+      - name: Create Installer
+        uses: joncloud/makensis-action@v1.2
+        with:
+          script-file: ./package/Tautulli.nsi
+          arguments: /DVERSION=${{ steps.get_version.outputs.VERSION_NSIS }} /DINSTALLER_NAME=..\Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}-x64.exe
+          include-more-plugins: true
+          include-custom-plugins-path: package/nsis-plugins
+
+      - name: Upload Installer
+        uses: actions/upload-artifact@v2
+        with:
+          name: Tautulli-windows-installer
+          path: Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}-x64.exe
+
+      - name: Post Status to Discord
+        uses: sarisia/actions-status-discord@v1
+        if: always()
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          status: ${{ job.status }}
+          title: Build Windows Installer
+          nofail: true
+
+  build-macos:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Set Release Version
+        id: get_version
+        shell: bash
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo ::set-env name=VERSION::${GITHUB_REF#refs/tags/v}
+            echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/v}
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+          else
+            echo ::set-env name=VERSION::0.0.0
+            echo ::set-output name=VERSION::0.0.0
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_SHA::7}
+          fi
+          echo $GITHUB_SHA > version.txt
+
+      - name: Set Up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Cache Dependencies
+        id: cache_dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/Library/Caches/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('package/requirements-macos.txt') }}
+          restore-keys: ${{ runner.os }}-pip-
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r package/requirements-macos.txt
+
+      - name: Build Package
+        run: |
+          pyinstaller -y ./package/Tautulli-macos.spec
+
+      - name: Create Installer
+        run: |
+          sudo pkgbuild --install-location /Applications --version ${{ steps.get_version.outputs.VERSION }} --component ./dist/Tautulli.app --scripts ./package/macos-scripts Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}-x64.pkg
+
+      - name: Upload Installer
+        uses: actions/upload-artifact@v2
+        with:
+          name: Tautulli-macos-installer
+          path: Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}-x64.pkg
+
+      - name: Post Status to Discord
+        uses: sarisia/actions-status-discord@v1
+        if: always()
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          status: ${{ job.status }}
+          title: Build MacOS Installer
+          nofail: true
+
+  release:
+    needs: [build-windows, build-macos]
+    if: startsWith(github.ref, 'refs/tags/') && always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get Build Job Status
+        uses: technote-space/workflow-conclusion-action@v1
+
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Set Release Version
+        id: get_version
+        run: |
+          echo ::set-output name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+
+      - name: Download Windows Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/download-artifact@v2
+        with:
+          name: Tautulli-windows-installer
+
+      - name: Download MacOS Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/download-artifact@v2
+        with:
+          name: Tautulli-macos-installer
+
       - name: Get Changelog
-        run: echo ::set-env name=CHANGELOG::"$( sed -n '/^## /{p; :loop n; p; /^## /q; b loop}' CHANGELOG.md | sed '$d' | sed '$d' | sed '$d' | sed ':a;N;$!ba;s/\n/%0A/g' )"
+        id: get_changelog
+        run: echo ::set-output name=CHANGELOG::"$( sed -n '/^## /{p; :loop n; p; /^## /q; b loop}' CHANGELOG.md | sed '$d' | sed '$d' | sed '$d' | sed ':a;N;$!ba;s/\n/%0A/g' )"
+
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          tag_name: ${{ env.RELEASE_VERSION }}
+          tag_name: ${{ steps.get_version.outputs.RELEASE_VERSION }}
-          release_name: Tautulli ${{ env.RELEASE_VERSION }}
+          release_name: Tautulli ${{ steps.get_version.outputs.RELEASE_VERSION }}
           body: |
             ## Changelog
             
-            ##${{ env.CHANGELOG }}
+            ##${{ steps.get_changelog.outputs.CHANGELOG }}
           draft: false
-          prerelease: ${{ endsWith(env.RELEASE_VERSION, '-beta') }}
+          prerelease: ${{ endsWith(steps.get_version.outputs.RELEASE_VERSION, '-beta') }}
+
+      - name: Upload Windows Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}-x64.exe
+          asset_name: Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}-x64.exe
+          asset_content_type: application/vnd.microsoft.portable-executable
+
+      - name: Upload MacOS Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}-x64.pkg
+          asset_name: Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}-x64.pkg
+          asset_content_type: application/vnd.apple.installer+xml

.gitignore

@@ -19,6 +19,8 @@ backups/*
 cache/*
 newsletters/*
 *.mmdb
+version.txt
+branch.txt
 
 # HTTPS Cert/Key #
 ##################
@@ -74,3 +76,7 @@ _ReSharper*/
 /logs
 .project
 .pydevproject
+
+#Ignore files generated by pyinstaller
+/build
+/dist

API.md

@@ -85,10 +85,10 @@ Delete all Tautulli history for a specific library.
 
 ```
 Required parameters:
+    server_id (str):        The Plex server identifier of the library section
     section_id (str):       The id of the Plex library section
 
 Optional parameters:
-    server_id (str):        The Plex server identifier of the library section
     row_ids (str):          Comma separated row ids to delete, e.g. "2,3,8"
 
 Returns:
@@ -159,10 +159,10 @@ Delete a library section from Tautulli. Also erases all history for the library.
 
 ```
 Required parameters:
+    server_id (str):        The Plex server identifier of the library section
     section_id (str):       The id of the Plex library section
 
 Optional parameters:
-    server_id (str):        The Plex server identifier of the library section
     row_ids (str):          Comma separated row ids to delete, e.g. "2,3,8"
 
 Returns:
@@ -427,6 +427,7 @@ Returns:
                  "children_count": "",
                  "collections": [],
                  "container": "mkv",
+                 "container_decision": "direct play",
                  "content_rating": "TV-MA",
                  "deleted_user": 0,
                  "device": "Windows",
@@ -1065,12 +1066,14 @@ Returns:
         [{"friendly_name": "Jon Snow",
           "total_plays": 170,
           "user_id": 133788,
-          "user_thumb": "https://plex.tv/users/k10w42309cynaopq/avatar"
+          "user_thumb": "https://plex.tv/users/k10w42309cynaopq/avatar",
+          "username": "LordCommanderSnow"
           },
          {"platform_type": "DanyKhaleesi69",
           "total_plays": 42,
           "user_id": 8008135,
-          "user_thumb": "https://plex.tv/users/568gwwoib5t98a3a/avatar"
+          "user_thumb": "https://plex.tv/users/568gwwoib5t98a3a/avatar",
+          "username: "DanyKhaleesi69"
           },
          {...},
          {...}
@@ -1180,6 +1183,7 @@ Returns:
          "grandparent_thumb": "/library/metadata/1219/thumb/1462175063",
          "grandparent_title": "Game of Thrones",
          "guid": "com.plexapp.agents.thetvdb://121361/6/1?lang=en",
+         "guids": [],
          "labels": [],
          "last_viewed_at": "1462165717",
          "library_name": "TV Shows",
@@ -1901,6 +1905,7 @@ Returns:
               "grandparent_thumb": "/library/metadata/1219/thumb/1462175063",
               "grandparent_title": "Game of Thrones",
               "guid": "com.plexapp.agents.thetvdb://121361/6/1?lang=en",
+              "guids": [],
               "labels": [],
               "last_viewed_at": "1462165717",
               "library_name": "TV Shows",
@@ -1989,6 +1994,33 @@ Returns:
 ```
 
 
+### get_server_info
+Get the PMS server information.
+
+```
+Required parameters:
+    None
+
+Optional parameters:
+    None
+
+Returns:
+    json:
+        {"pms_identifier": "08u2phnlkdshf890bhdlksghnljsahgleikjfg9t",
+         "pms_ip": "10.10.10.1",
+         "pms_is_remote": 0,
+         "pms_name": "Winterfell-Server",
+         "pms_platform": "Windows",
+         "pms_plexpass": 1,
+         "pms_port": 32400,
+         "pms_ssl": 0,
+         "pms_url": "http://10.10.10.1:32400",
+         "pms_url_manual": 0,
+         "pms_version": "1.20.0.3133-fede5bdc7"
+        }
+```
+
+
 ### get_server_list
 Get all your servers that are published to Plex.tv.
 
@@ -2267,8 +2299,8 @@ Required parameters:
     user_id (str):                  The id of the Plex user
 
 Optional parameters:
-    order_column (str):             "last_seen", "ip_address", "platform", "player",
+    order_column (str):             "last_seen", "first_seen", "ip_address", "platform",
-                                    "last_played", "play_count"
+                                    "player", "last_played", "play_count"
     order_dir (str):                "desc" or "asc"
     start (int):                    Row to start from, 0
     length (int):                   Number of items to return, 25
@@ -2286,6 +2318,7 @@ Returns:
               "ip_address": "xxx.xxx.xxx.xxx",
               "last_played": "Game of Thrones - The Red Woman",
               "last_seen": 1462591869,
+              "first_seen": 1583968210,
               "live": 0,
               "media_index": 1,
               "media_type": "episode",
@@ -2514,6 +2547,7 @@ Returns:
               "transcode_decision": "transcode",
               "user_id": 133788,
               "user_thumb": "https://plex.tv/users/568gwwoib5t98a3a/avatar",
+              "username": "LordCommanderSnow",
               "year": 2016
               },
              {...},
@@ -2554,20 +2588,53 @@ Returns:
 ```
 
 
-### import_database
+### import_config
-Import a PlexWatch or Plexivity database into Tautulli.
+Import a Tautulli config file.
 
 ```
 Required parameters:
-    app (str):                      "plexwatch" or "plexivity"
+    config_file (file):             The config file to import (multipart/form-data)
-    database_path (str):            The full path to the plexwatch database file
+    or
-    table_name (str):               "processed" or "grouped"
+    config_path (str):              The full path to the config file to import
+
 
 Optional parameters:
-    import_ignore_interval (int):   The minimum number of seconds for a stream to import
+    backup (bool):                  true or false whether to backup
+                                    the current config before importing
 
 Returns:
-    None
+    json:
+        {"result": "success",
+         "message": "Config import has started. Check the logs to monitor any problems. "
+                    "Tautulli will restart automatically."
+         }
+```
+
+
+### import_database
+Import a Tautulli, PlexWatch, or Plexivity database into Tautulli.
+
+```
+Required parameters:
+    app (str):                      "tautulli" or "plexwatch" or "plexivity"
+    database_file (file):           The database file to import (multipart/form-data)
+    or
+    database_path (str):            The full path to the database file to import
+    method (str):                   For Tautulli only, "merge" or "overwrite"
+    table_name (str):               For PlexWatch or Plexivity only, "processed" or "grouped"
+
+
+Optional parameters:
+    backup (bool):                  For Tautulli only, true or false whether to backup
+                                    the current database before importing
+    import_ignore_interval (int):   For PlexWatch or Plexivity only, the minimum number
+                                    of seconds for a stream to import
+
+Returns:
+    json:
+        {"result": "success",
+         "message": "Database import has started. Check the logs to monitor any problems."
+         }
 ```
 
 
@@ -2663,14 +2730,18 @@ Registers the Tautulli Android App for notifications.
 
 ```
 Required parameters:
-    device_name (str):        The device name of the Tautulli Android App
+    device_id (str):          The unique device identifier for the mobile device
-    device_id (str):          The OneSignal device id of the Tautulli Android App
+    device_name (str):        The device name of the mobile device
 
 Optional parameters:
     friendly_name (str):      A friendly name to identify the mobile device
+    onesignal_id (str):       The OneSignal id for the mobile device
 
 Returns:
-    None
+    json:
+        {"pms_name": "Winterfell-Server",
+         "server_id": "ds48g4r354a8v9byrrtr697g3g79w"
+         }
 ```
 
 

CHANGELOG.md

@@ -1,5 +1,119 @@
 # Changelog
 
+## v2.5.6 (2020-10-02)
+
+* Activity:
+  * Change: Renamed container "Transcode" to "Converting" on activity cards.
+* Notifications:
+  * New: Added a silent notification option for Telegram. (Thanks @JohnnyKing94)
+  * New: Added container_decision notification parameter.
+  * New: Added notification trigger for Playback Error.
+  * New: Added remote access down notification threshold setting.
+* Newsletter:
+  * Change: Stop flooring newsletter start date.
+* UI:
+  * Fix: Unable to purge history from the library edit modal.
+  * Fix: QR code not showing up for localhost address when trying to register a device.
+  * New: Added library name to the fix metadata modal.
+* API:
+  * New: Added default thumb and art to the Live TV library.
+* Other:
+  * Fix: Synced items not loading for guest access.
+  * New: Schedule some more automatic database optimizations.
+  * Change: Added automatic uninstall before installing to the Windows installer.
+
+
+## v2.5.5 (2020-09-06)
+
+* Activity:
+  * Fix: Filter out TV show background theme music sessions.
+* Notifications:
+  * New: Check Plex external guids for notification metadata provider links.
+* UI:
+  * Fix: Incorrect sorting for user/library recently played items.
+* API:
+  * Fix: get_synced_items API command returning error with empty result.
+  * Fix: Download API commands not returning the file.
+  * Fix: get_logs API command encoding error.
+  * Fix: get_user_player_stats API command returning error instead of empty result.
+  * New: Added get_server_info API command.
+  * New: Added external guids to get_metadata API command.
+  * New: Added support for multi-column sorting for datatable API commands.
+  * Change: get_activity API command return thumbnail override for clips.
+  * Change: get_libraries_table API command return custom library artwork.
+* Other:
+  * Fix: Tautulli failed to run with a stale pid file.
+  * New: Added scheduled task to optimize the Tautulli database.
+  * Change: Update plexapi to 3.6.0.
+  * Change: Update some libraries for Python 3 compatibility.
+
+
+## v2.5.4 (2020-07-31)
+
+* Monitoring:
+  * Change: Montitoring remote access changed to use websockets. Refer to Tautulli/Tautulli-Issues#251 for details.
+* Notifications:
+  * Fix: Uploading images to Cloudinary failed for titles with non-ASCII characters on Python 2.
+  * New: Added plex_id notification parameter.
+  * Remove: Running .exe files directly using script notifications is no longer supported.
+  * Remove: php, perl, and ruby prefix overrides for script notifications is no longer supported.
+  * Change: Stricter checking of file extensions for script notifications.
+  * Change: Fallback to The Movie Database lookup using title and year.
+  * Change: Fallback to TVmaze lookup using title.
+* UI:
+  * New: Added ability to import a previous Tautullli configuration file in the settings.
+  * New: Added a browse button for settings which require a folder or file input.
+  * New: Added first streamed column to user IP addresses table. (Thanks @dotsam)
+  * New: Added The Movie Database rating image to media page.
+  * Change: Different icon to represent direct stream in the history tables.
+* API:
+  * New: Updated API docs for importing a database and configuration file.
+
+
+## v2.5.3 (2020-07-10)
+
+* History:
+  * Fix: Unable to delete more than 1000 history entries at the same time.
+* Notifications:
+  * Change: Python script notifications to run using the same Python interpreter as Tautulli.
+* Newsletters:
+  * Fix: Unable to view newsletters with special characters.
+* Other:
+  * Fix: Tautulli failing to start after enabling HTTPS when installed using the Windows / macOS installers.
+  * Fix: Startup script not working on macOS.
+  * Fix: Unable to hide dock icon on macOS with the pkg install. Refer to the FAQ regarding the Python rocket dock icon.
+  * Change: Added path to Python interpreter in system startup (daemon) scripts.
+  * Change: Added Python version to Google analytics.
+
+
+## v2.5.2 (2020-07-01)
+
+* Announcements:
+  * Tautulli now supports Python 3!
+  * Python 2 is still supported for the time being, but it is recommended to upgrade to Python 3.
+* Notifications:
+  * Fix: Error uploading images to Cloudinary on Python 2.
+  * Fix: Testing browser notifications alert not disappearing.
+  * Change: Default recently added notification delay set to 300 seconds.
+* UI: 
+  * Fix: MacOS menu bar icon causing Tautulli to fail to start.
+  * Fix: Unable to login to Tautulli on Python 2.
+  * New: Windows and MacOS setting to enable Tautulli to start automatically when you login.
+  * New: Added menu bar icon for MacOS.
+  * New: Ability to import a Tautulli database in the settings.
+  * New: Added Tautulli news area on the settings page.
+  * New: Added platform icon for LG devices.
+  * Remove: Ability to login to Tautulli using a Plex username and password has been removed. Login using a Plex.tv account is only supported via OAuth.
+* Mobile App:
+  * Fix: Improved API security and validation when registering the Android app.
+* Docker:
+  * Fix: Docker container not respecting the PUID and PGID environment variables.
+* Other:
+  * Fix: Error creating self-signed certificates on Python 3.
+  * Fix: Tautulli login session cookie not set on the HTTP root path.
+  * New: Windows and MacOS app installers to install Tautulli without needing Python installed.
+
+
 ## v2.2.4 (2020-05-16)
 
 * Monitoring:

CONTRIBUTING.md

@@ -9,7 +9,7 @@ All pull requests should be based on the `nightly` branch, to minimize cross mer
 ### Python Code
 
 #### Compatibility
-The code should work with Python 2.7. Note that Tautulli runs on many different platforms.
+The code should work with Python 2.7.17 or Python 3.6+. Note that Tautulli runs on many different platforms.
 
 Re-use existing code. Do not hesitate to add logging in your code. You can the logger module `plexpy.logger.*` for this. Web requests are invoked via `plexpy.request.*` and derived ones. Use these methods to automatically add proper and meaningful error handling.
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM tautulli/tautulli-baseimage:latest
+FROM tautulli/tautulli-baseimage:python3
 
 LABEL maintainer="Tautulli"
 
@@ -16,7 +16,7 @@ RUN \
 
 COPY . /app
 
-CMD [ "python", "Tautulli.py", "--datadir", "/config" ]
+ENTRYPOINT [ "./start.sh" ]
 
 VOLUME /config
 EXPOSE 8181

PlexPy.py

@@ -1,8 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env python
-''''which python    >/dev/null 2>&1 && exec python    "$0" "$@" # '''
-''''which python2   >/dev/null 2>&1 && exec python2   "$0" "$@" # '''
-''''which python2.7 >/dev/null 2>&1 && exec python2.7 "$0" "$@" # '''
-''''exec echo "Error: Python not found!" # '''
 
 # -*- coding: utf-8 -*-
 

README.md

@@ -29,14 +29,15 @@ This project is based on code from [Headphones](https://github.com/rembo10/headp
 
 ## Installation & Support
 
-[![Python](https://img.shields.io/badge/python-v2.7.17-blue?style=flat-square)](https://python.org/downloads/release/python-2717/)
+[![Python](https://img.shields.io/badge/python-2.7.17,%203.6,%203.7,%203.8-blue?style=flat-square)](https://python.org/downloads)
 [![Docker Pulls](https://img.shields.io/docker/pulls/tautulli/tautulli?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli)
 [![Docker Stars](https://img.shields.io/docker/stars/tautulli/tautulli?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli)
 
 | Status | Branch: `master` | Branch: `beta` | Branch: `nightly` |
 | --- | --- | --- | --- |
 | Release   | [![Release@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![Release Date@master](https://img.shields.io/github/release-date/Tautulli/Tautulli?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/releases/latest) | [![Release@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![Commits@beta](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/beta?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/beta) | [![Last Commits@nightly](https://img.shields.io/github/last-commit/Tautulli/Tautulli/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) <br> [![Commits@nightly](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) |
-| Docker  | [![Docker@master](https://img.shields.io/badge/tautulli-tautulli:latest-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/master?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Amaster) | [![Docker@beta](https://img.shields.io/badge/tautulli-tautulli:beta-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@beta](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/beta?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Abeta) | [![Docker@nightly](https://img.shields.io/badge/tautulli-tautulli:nightly-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@nightly](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/nightly?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Anightly) |
+| Docker    | [![Docker@master](https://img.shields.io/badge/docker-latest-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/master?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Amaster) | [![Docker@beta](https://img.shields.io/badge/docker-beta-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@beta](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/beta?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Abeta) | [![Docker@nightly](https://img.shields.io/badge/docker-nightly-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@nightly](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/nightly?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Anightly) |
+| Installer | [![Windows@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=windows&style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![MacOS@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=macos&style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![Installer Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Release/master?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Release"+branch%3Amaster) | [![Windows@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=windows&include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![MacOS@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=macos&include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![Installer Build@beta](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Release/beta?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Release"+branch%3Abeta) | [![Installer Build@nightly](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Release/nightly?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Release"+branch%3Anightly) |
 
 [![Wiki](https://img.shields.io/badge/github-wiki-black?style=flat-square)](https://github.com/Tautulli/Tautulli-Wiki/wiki)
 [![Discord](https://img.shields.io/discord/183396325142822912?label=discord&style=flat-square&color=7289DA)](https://tautulli.com/discord)

Tautulli.py

@@ -1,8 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env python
-''''which python    >/dev/null 2>&1 && exec python    "$0" "$@" # '''
-''''which python2   >/dev/null 2>&1 && exec python2   "$0" "$@" # '''
-''''which python2.7 >/dev/null 2>&1 && exec python2.7 "$0" "$@" # '''
-''''exec echo "Error: Python not found!" # '''
 
 # -*- coding: utf-8 -*-
 
@@ -27,17 +23,24 @@ import sys
 # Ensure lib added to path, before any other imports
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib'))
 
+from future.builtins import str
+
+import appdirs
 import argparse
 import datetime
 import locale
 import pytz
 import signal
 import time
+import threading
 import tzlocal
 
 import plexpy
-from plexpy import config, database, helpers, logger, webstart
+from plexpy import common, config, database, helpers, logger, webstart
-
+if common.PLATFORM == 'Windows':
+    from plexpy import windows
+elif common.PLATFORM == 'Darwin':
+    from plexpy import macos
 
 # Register signals, such as CTRL + C
 signal.signal(signal.SIGINT, plexpy.sig_handler)
@@ -51,12 +54,14 @@ def main():
     """
 
     # Fixed paths to Tautulli
-    if hasattr(sys, 'frozen'):
+    if hasattr(sys, 'frozen') and hasattr(sys, '_MEIPASS'):
+        plexpy.FROZEN = True
         plexpy.FULL_PATH = os.path.abspath(sys.executable)
+        plexpy.PROG_DIR = sys._MEIPASS
     else:
         plexpy.FULL_PATH = os.path.abspath(__file__)
-
         plexpy.PROG_DIR = os.path.dirname(plexpy.FULL_PATH)
+
     plexpy.ARGS = sys.argv[1:]
 
     # From sickbeard
@@ -122,12 +127,11 @@ def main():
 
     if args.dev:
         plexpy.DEV = True
-        logger.debug(u"Tautulli is running in the dev environment.")
+        logger.debug("Tautulli is running in the dev environment.")
 
     if args.daemon:
         if sys.platform == 'win32':
-            sys.stderr.write(
+            logger.warn("Daemonizing not supported under Windows, starting normally")
-                "Daemonizing not supported under Windows, starting normally\n")
         else:
             plexpy.DAEMON = True
             plexpy.QUIET = True
@@ -145,11 +149,13 @@ def main():
             try:
                 with open(plexpy.PIDFILE, 'r') as fp:
                     pid = int(fp.read())
-                os.kill(pid, 0)
             except IOError as e:
                 raise SystemExit("Unable to read PID file: %s", e)
+
+            try:
+                os.kill(pid, 0)
             except OSError:
-                logger.warn("PID file '%s' already exists, but PID %d is " \
+                logger.warn("PID file '%s' already exists, but PID %d is "
                             "not running. Ignoring PID file." %
                             (plexpy.PIDFILE, pid))
             else:
@@ -175,6 +181,8 @@ def main():
     # Determine which data directory and config file to use
     if args.datadir:
         plexpy.DATA_DIR = args.datadir
+    elif plexpy.FROZEN:
+        plexpy.DATA_DIR = appdirs.user_data_dir("Tautulli", False)
     else:
         plexpy.DATA_DIR = plexpy.PROG_DIR
 
@@ -229,25 +237,51 @@ def main():
         try:
             import OpenSSL
         except ImportError:
-            logger.warn("The pyOpenSSL module is missing. Install this " \
+            logger.warn("The pyOpenSSL module is missing. Install this "
                         "module to enable HTTPS. HTTPS will be disabled.")
             plexpy.CONFIG.ENABLE_HTTPS = False
 
     # Try to start the server. Will exit here is address is already in use.
     webstart.start()
 
-    # Windows system tray icon
+    if common.PLATFORM == 'Windows':
-    if os.name == 'nt' and plexpy.CONFIG.WIN_SYS_TRAY:
+        if plexpy.CONFIG.SYS_TRAY_ICON:
-        plexpy.win_system_tray()
+            plexpy.WIN_SYS_TRAY_ICON = windows.WindowsSystemTray()
-
+            plexpy.WIN_SYS_TRAY_ICON.start()
-    logger.info("Tautulli is ready!")
+        windows.set_startup()
+    elif common.PLATFORM == 'Darwin':
+        macos.set_startup()
 
     # Open webbrowser
     if plexpy.CONFIG.LAUNCH_BROWSER and not args.nolaunch and not plexpy.DEV:
         plexpy.launch_browser(plexpy.CONFIG.HTTP_HOST, plexpy.HTTP_PORT,
                               plexpy.HTTP_ROOT)
 
-    # Wait endlessy for a signal to happen
+    if common.PLATFORM == 'Darwin' and plexpy.CONFIG.SYS_TRAY_ICON:
+        if not macos.HAS_PYOBJC:
+            logger.warn("The pyobjc module is missing. Install this "
+                        "module to enable the MacOS menu bar icon.")
+            plexpy.CONFIG.SYS_TRAY_ICON = False
+
+        if plexpy.CONFIG.SYS_TRAY_ICON:
+            # MacOS menu bar icon must be run on the main thread and is blocking
+            # Start the rest of Tautulli on a new thread
+            thread = threading.Thread(target=wait)
+            thread.daemon = True
+            thread.start()
+
+            plexpy.MAC_SYS_TRAY_ICON = macos.MacOSSystemTray()
+            plexpy.MAC_SYS_TRAY_ICON.start()
+        else:
+            wait()
+    else:
+        wait()
+
+
+def wait():
+    logger.info("Tautulli is ready!")
+
+    # Wait endlessly for a signal to happen
     while True:
         if not plexpy.SIGNAL:
             try:
@@ -265,11 +299,14 @@ def main():
                 plexpy.shutdown(restart=True, checkout=True)
             elif plexpy.SIGNAL == 'reset':
                 plexpy.shutdown(restart=True, reset=True)
-            else:
+            elif plexpy.SIGNAL == 'update':
                 plexpy.shutdown(restart=True, update=True)
+            else:
+                logger.error('Unknown signal. Shutting down...')
+                plexpy.shutdown()
 
             plexpy.SIGNAL = None
 
-# Call main()
+
 if __name__ == "__main__":
     main()

contrib/clean_pyc.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Display information
 echo "This script will remove *.pyc files. These files are generated by Python, but they can cause conflicts after an upgrade. It's safe to remove them, because they will be regenerated."

contrib/downgrade.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Parameter check
 if [ -z "$1" ]; then

data/interfaces/default/app_import.html

@@ -5,6 +5,9 @@
             <h4 class="modal-title">Import ${app} Database</h4>
         </div>
         <div class="modal-body" id="modal-text">
+            <form id="import_database_form" enctype="multipart/form-data" method="post" name="import_database_form">
+                <input type="hidden" id="import_app" name="import_app" value="${app.lower()}" />
+                % if app in ('PlexWatch', 'Plexivity'):
                 <p class="help-block">
                     <%
                         v = ''
@@ -15,26 +18,80 @@
                     %>
                     <strong>Please ensure your ${app} database is at version ${v} or higher.</strong>
                 </p>
+                % endif
                 <div class="form-group">
-                <label for="db_location">Database Location</label>
+                    <label for="import_database_file">Option 1: Upload a Database File</label>
                     <div class="row">
-                    <div class="col-xs-8">
+                        <div class="col-xs-12">
-                        <input type="text" class="form-control" id="db_location" name="db_location" value="" required>
+                            <div class="input-group">
+                                <label for="import_database_file" class="input-group-btn">
+                                    <span class="btn btn-form">Upload</span>
+                                    <input type="file" style="display: none;" id="import_database_file" name="import_database_file" required>
+                                </label>
+                                <input id="import_database_file_name" type="text" class="form-control" placeholder="tautulli.db" disabled>
                             </div>
                         </div>
-                <p class="help-block">Enter the path and file name for the ${app} database you wish to import.</p>
+                    </div>
+                    <p class="help-block">Upload the ${app} database file you wish to import.</p>
                 </div>
                 <div class="form-group">
-                <label for="table_name">Table Name</label>
+                    <label for="import_database_path">Option 2: Browse for a Database File</label>
+                    <div class="row">
+                        <div class="col-xs-12">
+                            <div class="input-group">
+                                <span class="input-group-btn">
+                                    <button class="btn btn-form" type="button" id="import_database_path_browse" data-toggle="browse" data-description="Database File" data-filter=".db" data-target="#import_database_path">Browse</button>
+                                </span>
+                                <input type="text" class="form-control" id="import_database_path" name="import_database_path" value="" placeholder="tautulli.db" required disabled>
+                            </div>
+                        </div>
+                    </div>
+                    <p class="help-block">Browse for the ${app} database file you wish to import.</p>
+                </div>
+                % if app == 'Tautulli':
+                <div class="form-group">
+                    <label for="table_name">Import Method</label>
                     <div class="row">
                         <div class="col-xs-4">
-                        <select id="table_name" class="form-control" name="table_name">
+                            <select class="form-control" id="import_method" name="import_method">
-                            <option value="processed">processed</option>
+                                <option value="merge">Merge</option>
-                            <option value="grouped">grouped</option>
+                                <option value="overwrite">Overwrite</option>
                             </select>
                         </div>
                     </div>
-                <p class="help-block">The table name from which you wish to import. Only import one of these, importing both will result in duplicated data.</p>
+                    <p class="help-block">Select how you would like to import the Tautulli history.</p>
+                    <ul class="help-block" style="padding-inline-start: 15px;">
+                        <li><strong>Merge</strong> will add all history and remove any duplicates from the imported database into the current database.</li>
+                        <li><strong>Overwrite</strong> will replace all history in the current database with the imported database.</li>
+                    </ul>
+                </div>
+                <div class="checkbox">
+                    <label>
+                        <input type="checkbox" name="import_backup_db" id="import_backup_db" value="1" checked> Backup Current Database
+                    </label>
+                    <p class="help-block">Automatically create a backup of the current database before importing.</p>
+                </div>
+                <div class="form-group">
+                    <label>Import Notes</label>
+                    <p class="help-block">The following data will also be imported:</p>
+                    <ul class="help-block" style="padding-inline-start: 15px;">
+                        <li>Libraries and Users</li>
+                        <li>Notification / Newsletter Agents</li>
+                        <li>Registered Mobile Devices</li>
+                    </ul>
+                </div>
+                % else:
+                <div class="form-group">
+                    <label for="import_table_name">Table Name</label>
+                    <div class="row">
+                        <div class="col-xs-4">
+                            <select class="form-control" id="import_table_name" name="import_table_name">
+                                <option value="processed">Processed</option>
+                                <option value="grouped">Grouped</option>
+                            </select>
+                        </div>
+                    </div>
+                    <p class="help-block">Select the table name from which you wish to import. Only import one of these, importing both will result in duplicated data.</p>
                 </div>
                 <div class="form-group">
                     <label for="import_ignore_interval">Ignore Interval</label>
@@ -45,6 +102,8 @@
                     </div>
                     <p class="help-block">Enter the minimum duration (in seconds) an item must have been active for. Set to 0 to import all.</p>
                 </div>
+                % endif
+            </form>
         </div>
         <div class="modal-footer">
             <div>
@@ -55,24 +114,87 @@
     </div>
 </div>
 <script>
-    // Send database path to import script
+    $("#import_database_file").change(function() {
+        if ($(this)[0].files[0]) {
+            $("#import_database_file_name").val($(this)[0].files[0].name);
+        }
+    });
+
     $("#import_db").click(function() {
-        var database_path = $("#db_location").val();
+        $(this).prop('disabled', true);
-        var table_name = $("#table_name").val();
+
-        var import_ignore_interval = $("#import_ignore_interval").val();
+        var app = $("#import_app").val();
+        var database_file = $("#import_database_file")[0].files[0];
+        var database_path = $("#import_database_path").val();
+        var method = $("#import_method").val();
+        var backup = $("#import_backup_db").is(':checked');
+        var table_name = $("#import_table_name").val();
+        var ignore_interval = $("#import_ignore_interval").val();
+
+        var content_type;
+        var process_data;
+        var data;
+
+        if (database_file) {
+            content_type = false;
+            process_data = false;
+            data = new FormData();
+            data.append('app', app);
+            data.append('database_file', database_file);
+            data.append('method', method);
+            data.append('backup', backup);
+            data.append('table_name', table_name);
+            data.append('ignore_interval', ignore_interval);
+        } else {
+            content_type = 'application/x-www-form-urlencoded; charset=UTF-8';
+            process_data = true;
+            data = {
+                app: app,
+                database_path: database_path,
+                method: method,
+                backup: backup,
+                table_name: table_name,
+                ignore_interval: ignore_interval
+            }
+        }
+
+        if (database_file) {
+            $("#status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>&nbsp; Uploading database file...');
+        } else {
+            $("#status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>');
+        }
+
         $.ajax({
             url: 'import_database',
-            data: {
+            type: 'POST',
-                app: "${app}",
+            data: data,
-                database_path: database_path,
-                table_name: table_name,
-                import_ignore_interval: import_ignore_interval
-            },
             cache: false,
             async: true,
+            contentType: content_type,
+            processData: process_data,
             success: function(data) {
-                $("#status-message").html(data);
+                var msg;
-                $("#db_location").val('')
+                if (data.result === 'success') {
+                    msg = "<i class='fa fa-check'></i>&nbsp; " + data.message;
+                } else {
+                    msg = "<i class='fa fa-exclamation-triangle'></i>&nbsp; " + data.message;
+                }
+                $("#status-message").html(msg);
+                $("#import_database_file").val(null);
+                $("#import_database_file_name").val('');
+                $("#import_database_path").val('');
+            },
+            error: function (xhr) {
+                var msg = "<i class='fa fa-exclamation-triangle'></i>&nbsp; Error (" + xhr.status + "): ";
+                if (xhr.status === 413) {
+                    msg += "file is too large to upload"
+                } else {
+                    msg += 'try again'
+                }
+                $("#status-message").html(msg);
+            },
+            complete: function(xhr) {
+                $("#import_db").prop('disabled', false);
             }
         });
     });

data/interfaces/default/base.html

@@ -55,8 +55,10 @@
         newer version</a> of Tautulli is available!<br />
         You are ${plexpy.COMMITS_BEHIND} commit${'s' if plexpy.COMMITS_BEHIND > 1 else ''} behind.<br />
         % endif
-        % if plexpy.DOCKER:
+        % if plexpy.INSTALL_TYPE == 'docker':
         Update your Docker container or <a href="#" id="updateDismiss">Dismiss</a>
+        % elif plexpy.INSTALL_TYPE in ('windows', 'macos'):
+        <a href="${anon_url('https://github.com/%s/%s/releases/tag/%s' % (plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO, plexpy.LATEST_RELEASE))}" target="_blank">Download</a> and install the latest version or <a href="#" id="updateDismiss">Dismiss</a>
         % else:
         <a href="update">Update</a> or <a href="#" id="updateDismiss">Dismiss</a>
         % endif
@@ -228,20 +230,12 @@ ${next.modalIncludes()}
                     </div>
                 </div>
                 <ul id="donation_type" class="nav nav-pills" role="tablist" style="display: flex; justify-content: center; margin: 10px 0;">
-                    <li class="active"><a href="#patreon-donation" role="tab" data-toggle="tab">Patreon</a></li>
+                    <li class="active"><a href="#github-donation" role="tab" data-toggle="tab">GitHub</a></li>
-                    <li><a href="#github-donation" role="tab" data-toggle="tab">GitHub</a></li>
+                    <li><a href="#patreon-donation" role="tab" data-toggle="tab">Patreon</a></li>
                     <li><a href="#paypal-donation" role="tab" data-toggle="tab">PayPal</a></li>
                 </ul>
                 <div class="tab-content">
-                    <div role="tabpanel" class="tab-pane active" id="patreon-donation" style="text-align: center">
+                    <div role="tabpanel" class="tab-pane active" id="github-donation" style="text-align: center">
-                        <p>
-                            Click the button below to continue to Patreon.
-                        </p>
-                        <a href="${anon_url('https://www.patreon.com/join/tautulli')}" target="_blank">
-                            <img src="images/become_a_patron_button.png" alt="Become a Patron" height="40">
-                        </a>
-                    </div>
-                    <div role="tabpanel" class="tab-pane" id="github-donation" style="text-align: center">
                         <p>
                             Click the button below to continue to GitHub.
                         </p>
@@ -249,6 +243,14 @@ ${next.modalIncludes()}
                             <i class="fa fa-heart fa-sm" style="color: #ea4aaa;"></i>&nbsp; Sponsor
                         </a>
                     </div>
+                    <div role="tabpanel" class="tab-pane" id="patreon-donation" style="text-align: center">
+                        <p>
+                            Click the button below to continue to Patreon.
+                        </p>
+                        <a href="${anon_url('https://www.patreon.com/join/tautulli')}" target="_blank">
+                            <img src="images/become_a_patron_button.png" alt="Become a Patron" height="40">
+                        </a>
+                    </div>
                     <div role="tabpanel" class="tab-pane" id="paypal-donation" style="text-align: center">
                         <p>
                             Click the button below to continue to PayPal.
@@ -294,9 +296,7 @@ ${next.modalIncludes()}
 <script src="${http_root}js/ipaddr.min.js"></script>
 <script src="${http_root}js/script.js${cache_param}"></script>
 <script src="${http_root}js/jquery.tripleclick.min.js"></script>
-% if _session['user_group'] == 'admin' and BROWSER_NOTIFIERS:
 <script src="${http_root}js/ajaxNotifications.js"></script>
-% endif
 <script>
     % if _session['user_group'] == 'admin':
     $('body').on('click', '#updateDismiss', function() {
@@ -330,8 +330,10 @@ ${next.modalIncludes()}
                         msg = 'A <a href="' + result.compare_url + '" target="_blank">newer version</a> of Tautulli is available!<br />' +
                             'You are '+ result.commits_behind + ' commit' + (result.commits_behind > 1 ? 's' : '') + ' behind.<br />';
                     }
-                    if (result.docker) {
+                    if (result.install_type === 'docker') {
                         msg += 'Update your Docker container or <a href="#" id="updateDismiss">Dismiss</a>';
+                    } else if (result.install_type === 'windows' || result.install_type === 'macos') {
+                        msg += '<a href="' + result.release_url + '" target="_blank">Download</a> and install the latest version or <a href="#" id="updateDismiss">Dismiss</a>'
                     } else {
                         msg += '<a href="update">Update</a> or <a href="#" id="updateDismiss">Dismiss</a>';
                     }
@@ -419,6 +421,10 @@ ${next.modalIncludes()}
         $(document).on('hidden.bs.modal', '.modal', function () {
             $('.modal:visible').length && $(document.body).addClass('modal-open');
         });
+
+        % if _session['user_group'] == 'admin' and BROWSER_NOTIFIERS:
+        check_notifications();
+        % endif
     });
 
     % if _session['user_group'] != 'admin':

data/interfaces/default/config_import.html

@@ -0,0 +1,138 @@
+<div class="modal-dialog" role="document">
+    <div class="modal-content">
+        <div class="modal-header">
+            <button type="button" class="close" data-dismiss="modal" aria-hidden="true"><i class="fa fa-remove"></i></button>
+            <h4 class="modal-title">${title}</h4>
+        </div>
+        <div class="modal-body" id="modal-text">
+            <form id="import_config_form" enctype="multipart/form-data" method="post" name="import_config_form">
+                <div class="form-group">
+                    <label for="import_config_file">Option 1: Upload a Configuration File</label>
+                    <div class="row">
+                        <div class="col-xs-12">
+                            <div class="input-group">
+                                <label for="import_config_file" class="input-group-btn">
+                                    <span class="btn btn-form">Upload</span>
+                                    <input type="file" style="display: none;" id="import_config_file" name="import_config_file" required>
+                                </label>
+                                <input id="import_config_file_name" type="text" class="form-control" placeholder="config.ini" disabled>
+                            </div>
+                        </div>
+                    </div>
+                    <p class="help-block">Upload the Tautulli configuration file you wish to import.</p>
+                </div>
+                <div class="form-group">
+                    <label for="import_config_path">Option 2: Browse for a Configuration File</label>
+                    <div class="row">
+                        <div class="col-xs-12">
+                            <div class="input-group">
+                                <span class="input-group-btn">
+                                    <button class="btn btn-form" type="button" id="import_config_path_browse" data-toggle="browse" data-description="Configuration File" data-filter=".ini" data-target="#import_config_path">Browse</button>
+                                </span>
+                                <input type="text" class="form-control" id="import_config_path" name="import_config_path" value="" placeholder="config.ini" required disabled>
+                            </div>
+                        </div>
+                    </div>
+                    <p class="help-block">Browse for the Tautulli configuration file you wish to import.</p>
+                </div>
+                <div class="checkbox">
+                    <label>
+                        <input type="checkbox" name="import_backup_config" id="import_backup_config" value="1" checked> Backup Current Configuration
+                    </label>
+                    <p class="help-block">Automatically create a backup of the current configuration before importing.</p>
+                </div>
+                <div class="form-group">
+                    <label>Import Notes</label>
+                    <p class="help-block">The following settings will <em>not</em> be imported:</p>
+                    <ul class="help-block" style="padding-inline-start: 15px;">
+                        <li>Git Path, Log / Backup / Cache Directory, Plex Logs Folder</li>
+                        <li>Custom Newsletter Templates Folder, Newsletter Output Directory</li>
+                        <li>HTTP Host / Port / Root / Username / Password</li>
+                        <li>Enable HTTPS, HTTPS Certificate / Certificate Chain / Key</li>
+                    </ul>
+                </div>
+            </form>
+        </div>
+        <div class="modal-footer">
+            <div>
+                <span id="status-message" style="padding-right: 25px;"></span>
+                <input type="button" id="import_config" class="btn btn-bright" value="Import">
+            </div>
+        </div>
+    </div>
+</div>
+<script>
+    $("#import_config_file").change(function() {
+        if ($(this)[0].files[0]) {
+            $("#import_config_file_name").val($(this)[0].files[0].name);
+        }
+    });
+
+    $("#import_config").click(function() {
+        $(this).prop('disabled', true);
+
+        var config_file = $("#import_config_file")[0].files[0];
+        var config_path = $("#import_config_path").val();
+        var backup = $("#import_backup_config").is(':checked');
+
+        var content_type;
+        var process_data;
+        var data;
+
+        if (config_file) {
+            content_type = false;
+            process_data = false;
+            data = new FormData();
+            data.append('config_file', config_file);
+            data.append('backup', backup);
+        } else {
+            content_type = 'application/x-www-form-urlencoded; charset=UTF-8';
+            process_data = true;
+            data = {
+                config_path: config_path,
+                backup: backup
+            }
+        }
+
+        if (config_file) {
+            $("#status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>&nbsp; Uploading config file...');
+        } else {
+            $("#status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>');
+        }
+
+        $.ajax({
+            url: 'import_config',
+            type: 'POST',
+            data: data,
+            cache: false,
+            async: true,
+            contentType: content_type,
+            processData: process_data,
+            success: function(data) {
+                var msg;
+                if (data.result === 'success') {
+                    msg = "<i class='fa fa-check'></i>&nbsp; " + data.message;
+                    window.location.href = 'restart_import_config';
+                } else {
+                    msg = "<i class='fa fa-exclamation-triangle'></i>&nbsp; " + data.message;
+                }
+                $("#status-message").html(msg);
+                $("#import_config_file").val(null);
+                $("#import_config_file_name").val('');
+                $("#import_config_path").val('');
+            },
+            error: function (xhr) {
+                var msg = "<i class='fa fa-exclamation-triangle'></i>&nbsp; Error (" + xhr.status + "): ";
+                if (xhr.status === 413) {
+                    msg += "file is too large to upload"
+                } else {
+                    msg += 'try again'
+                }
+                $("#status-message").html(msg);
+            },
+            complete: function(xhr) {
+                $("#import_config").prop('disabled', false);
+            }
+        });
+    });
+</script>

data/interfaces/default/css/tautulli.css

@@ -1,6 +1,6 @@
 body {
     font-family: 'Open Sans', Arial, sans-serif;
-    color: #fff;
+    color: #eee;
     margin-top: 50px;
     overflow: hidden;
 }
@@ -36,7 +36,7 @@ select.input-sm {
 select[multiple] {
     height: 125px;
     margin: 5px 0 5px 0;
-    color: #fff;
+    color: #eee;
     border: 0px solid #444;
     background: #555;
     padding: 2px 2px;
@@ -48,7 +48,7 @@ select[multiple]:focus {
     outline: 0;
     outline: thin dotted \9;
     color: #555;
-    background-color: #fff;
+    background-color: #eee;
     transition: background-color .3s;
 }
 select[multiple]:focus::-webkit-scrollbar-thumb {
@@ -63,7 +63,7 @@ select[multiple] option {
 select.form-control,
 div.form-control .selectize-input {
     margin: 5px 0 5px 0;
-    color: #fff;
+    color: #eee;
     border: 0px solid #444;
     background: #555;
     padding: 6px 12px;
@@ -76,7 +76,7 @@ select.form-control {
 }
 .react-selectize.root-node .react-selectize-control,
 .selectize-control.form-control .selectize-input {
-    color: #fff !important;
+    color: #eee !important;
     border: 0px solid #444 !important;
     background: #555 !important;
     padding: 1px 2px;
@@ -123,15 +123,15 @@ select.form-control {
     cursor: pointer;
 }
 .react-selectize.root-node .react-selectize-control .react-selectize-placeholder {
-    color: #fff !important;
+    color: #eee !important;
 }
 .react-selectize.root-node .react-selectize-control .react-selectize-toggle-button path {
-    fill: #fff !important;
+    fill: #eee !important;
 }
 .react-selectize.root-node .simple-value,
 .selectize-control.multi .selectize-input > div {
-    background: #444444 !important;
+    background: #444 !important;
-    color: #ffffff !important;
+    color: #eee !important;
     padding-bottom: 2px !important;
     transition: background-color .3s;
 }
@@ -156,7 +156,7 @@ select.form-control:focus,
     outline: 0;
     outline: thin dotted \9;
     color: #555 !important;
-    background-color: #fff !important;
+    background-color: #eee !important;
     transition: background-color .3s;
 }
 .react-selectize.root-node.open .simple-value,
@@ -219,7 +219,7 @@ select.form-control:focus,
 }
 select.form-control option {
     color: #555;
-    background-color: #fff;
+    background-color: #eee;
 }
 img {
     -webkit-box-sizing: content-box;
@@ -278,13 +278,13 @@ object {
 }
 .dropdown-menu > li > a:hover,
 .dropdown-menu > li > a:focus {
-  color: #fff;
+  color: #eee;
   background-color: #2f2f2f;
 }
 .dropdown-menu > .active > a,
 .dropdown-menu > .active > a:hover,
 .dropdown-menu > .active > a:focus {
-  color: #fff;
+  color: #eee;
   background-color: #2f2f2f;
 }
 .dropdown-menu > .disabled > a,
@@ -327,14 +327,14 @@ object {
   background-color: #3B3B3B;
 }
 .btn-dark:hover {
-  color: #fff;
+  color: #eee;
   background-color: #333;
   border-color: #444;
 }
 .btn-dark:active,
 .btn-dark.active,
 .open > .dropdown-toggle.btn-dark {
-  color: #fff;
+  color: #eee;
   background-color: #333;
   border-color: #444;
 }
@@ -347,7 +347,7 @@ object {
 .btn-dark:active.focus,
 .btn-dark.active.focus,
 .open > .dropdown-toggle.btn-dark.focus {
-  color: #fff;
+  color: #eee;
   background-color: #333;
 }
 .btn-dark:active,
@@ -387,24 +387,24 @@ fieldset[disabled] .btn-dark.active {
   background-color: #3B3B3B;
 }
 .btn-bright {
-  color: #fff;
+  color: #eee;
   background-color: #cc7b19;
   box-shadow: inset 0 1px 0 #e7993b;
 }
 .btn-bright:focus,
 .btn-bright.focus {
-  color: #fff;
+  color: #eee;
   background-color: #eb8600;
 }
 .btn-bright:hover {
-  color: #fff;
+  color: #eee;
   background-color: #e59029;
   box-shadow: inset 0 1px 0 #ebac60;
 }
 .btn-bright:active,
 .btn-bright.active,
 .open > .dropdown-toggle.btn-bright {
-  color: #fff;
+  color: #eee;
   background-color: #cc7b19;
   box-shadow: inset 0 1px 0 #e7993b;
 }
@@ -417,7 +417,7 @@ fieldset[disabled] .btn-dark.active {
 .btn-bright:active.focus,
 .btn-bright.active.focus,
 .open > .dropdown-toggle.btn-bright.focus {
-  color: #fff;
+  color: #eee;
   background-color: #cc7b19;
   box-shadow: inset 0 1px 0 #e7993b;
 }
@@ -448,7 +448,7 @@ fieldset[disabled] .btn-bright.active {
   border-color: #b56d16;
 }
 .btn-bright .badge {
-  color: #fff;
+  color: #eee;
   background-color: #cc7b19;
   box-shadow: inset 0 1px 0 #e7993b;
 }
@@ -459,22 +459,26 @@ fieldset[disabled] .btn-bright.active {
     float: right;
 }
 .btn-danger.btn-edit:hover {
-    color: #fff;
+    color: #eee;
     background-color: #c9302c;
     border-color: #ac2925;
 }
 .btn-danger.btn-edit.active {
-  color: #fff;
+  color: #eee;
   background-color: #c9302c;
   border-color: #ac2925;
 }
 .btn-danger.btn-edit.active:hover {
-  color: #fff;
+  color: #eee;
   background-color: #ac2925;
   border-color: #761c19;
 }
 .btn-group select {
     margin-top: 0;
+    height: 34px;
+}
+.btn-group label {
+    margin-bottom: 0;
 }
 .input-group-addon-form {
     display: inline-block;
@@ -488,9 +492,6 @@ fieldset[disabled] .btn-bright.active {
     width: 100%;
     margin-top: 5px;
 }
-#user-selection label {
-    margin-bottom: 0;
-}
 .alert-edit {
     display: none;
     float: left;
@@ -512,7 +513,7 @@ fieldset[disabled] .btn-bright.active {
     background-color: #222222;
 }
 .modal-body table {
-    color: #fff;
+    color: #eee;
 }
 .modal-body li {
     margin-top: 7px;
@@ -526,7 +527,7 @@ fieldset[disabled] .btn-bright.active {
     color: #E5A00D;
 }
 .modal-body i.fa {
-    color: #fff;
+    color: #eee;
 }
 .modal-body td:hover a .fa,
 .modal-body a:focus i.fa {
@@ -560,7 +561,7 @@ input[type="tel"],
 input[type="color"],
 .uneditable-input {
     margin: 5px 0 5px 0;
-    color: #fff;
+    color: #eee;
     border: 0px solid #444;
     background: #555;
     height: 32px;
@@ -572,7 +573,7 @@ input[type="color"],
 textarea.form-control {
     height: initial;
     margin: 5px 0 5px 0;
-    color: #fff;
+    color: #eee;
     border: 0px solid #444;
     background: #555;
     padding: 6px 12px;
@@ -584,7 +585,7 @@ textarea.form-control {
 textarea.form-control:focus {
     outline: 0;
     color: #555;
-    background-color: #fff;
+    background-color: #eee;
     transition: background-color .3s;
 }
 .pagination > li > a,
@@ -594,7 +595,7 @@ textarea.form-control:focus {
     padding: 6px 12px;
     margin-left: -1px;
     line-height: 1.42857143;
-    color: #fff;
+    color: #eee;
     text-decoration: none;
     background-color: #262626;
     border: 1px solid #444444;
@@ -613,7 +614,7 @@ textarea.form-control:focus {
 .pagination > .active > a:focus,
 .pagination > .active > span:focus {
     z-index: 2;
-    color: #fff;
+    color: #eee;
     cursor: default;
     background-color: #cc7b19;
     border-color: #444444;
@@ -632,7 +633,7 @@ textarea.form-control:focus {
 .nav-pills > li.active > a,
 .nav-pills > li.active > a:hover,
 .nav-pills > li.active > a:focus {
-    color: #fff;
+    color: #eee;
     background-color: #cc7b19;
 }
 .nav-pills > li > a {
@@ -666,11 +667,11 @@ textarea.form-control:focus {
     -webkit-appearance:none;
 }
 .btn-form:hover {
-    color: #fff;
+    color: #eee;
     background-color: #333;
 }
 .btn-form:focus {
-    color: #fff;
+    color: #eee;
 }
 .form-control-feedback {
     color: #E5A00D;
@@ -682,7 +683,7 @@ fieldset[disabled] .form-control {
     background-color: #555;
 }
 .form-control[readonly]:focus {
-    background-color: #fff;
+    background-color: #eee;
 }
 .poster {
     position: relative;
@@ -1071,7 +1072,7 @@ a:hover .dashboard-activity-cover {
     font-size: 13px;
     font-weight: bold;
     line-height: 25px;
-    color: #fff;
+    color: #eee;
 }
 .dashboard-activity-metadata-play_state-icon {
     flex-basis: 25px;
@@ -1534,7 +1535,7 @@ a:hover .dashboard-recent-media-cover {
 }
 .dashboard-recent-media-metacontainer h3 {
     padding: 5px 3px 0 3px;
-    color: #fff;
+    color: #eee;
     text-overflow: ellipsis;
     overflow: hidden;
     position: relative;
@@ -1647,12 +1648,12 @@ a:hover .dashboard-recent-media-cover {
     color: #f9be03;
 }
 .summary-content-title h1 a:hover {
-    color: #fff;
+    color: #eee;
 }
 .summary-content-title h2 {
     margin-top: 0;
     margin-bottom: 10px;
-    color: #fff;
+    color: #eee;
     font-size: 28px;
     line-height: 40px;
     float: left;
@@ -1806,7 +1807,7 @@ a:hover .summary-poster-face-track .summary-poster-face-overlay span {
     line-height: 24px;
 }
 .summary-content-details-tag strong {
-    color: #fff;
+    color: #eee;
     margin-left: 2px;
     margin-right: 10px;
 }
@@ -1826,7 +1827,7 @@ a:hover .summary-poster-face-track .summary-poster-face-overlay span {
 }
 .summary-content-summary {
     overflow: hidden;
-    color: #fff;
+    color: #eee;
     float: left;
     position: relative;
     clear: both;
@@ -1860,7 +1861,7 @@ a:hover .summary-poster-face-track .summary-poster-face-overlay span {
     display: block;
     font-size: 12px;
     line-height: 18px;
-    color: #fff;
+    color: #eee;
 }
 .summary-content-genres {
     margin-top: 13px;
@@ -1879,7 +1880,7 @@ a:hover .summary-poster-face-track .summary-poster-face-overlay span {
     display: block;
     font-size: 12px;
     line-height: 18px;
-    color: #fff;
+    color: #eee;
 }
 .summary-content-writers {
     margin-top: 13px;
@@ -1898,7 +1899,7 @@ a:hover .summary-poster-face-track .summary-poster-face-overlay span {
     display: block;
     font-size: 12px;
     line-height: 18px;
-    color: #fff;
+    color: #eee;
 }
 .star-rating {
     display: inline-block;
@@ -1951,7 +1952,7 @@ a:hover .summary-poster-face-track .summary-poster-face-overlay span {
     position: relative;
     margin: 0;
     line-height: 22px;
-    color: #fff;
+    color: #eee;
     font-size: 16px;
     text-align: center;
     text-transform: uppercase;
@@ -2047,7 +2048,7 @@ a:hover .item-children-poster {
 .item-children-instance-text-wrapper h3 {
     width: 100%;
     padding: 5px 3px 0 3px;
-    color: #fff;
+    color: #eee;
     text-overflow: ellipsis;
     overflow: hidden;
     position: relative;
@@ -2112,7 +2113,7 @@ a:hover .item-children-poster {
     margin-right: 20px;
 }
 #new_title h3 {
-    color: #f9be03;
+    color: #E5A00D;
     font-size: 14px;
     line-height: 1.42857143;
     font-weight: bold;
@@ -2148,7 +2149,7 @@ span.settings-warning {
     padding-left: 10px;
 }
 #menu_link_show_advanced_settings.active {
-  color: #fff;
+  color: #eee;
   background-color: #cc7b19;
 }
 .advanced-setting {
@@ -2161,7 +2162,7 @@ div.advanced-setting {
 li.advanced-setting {
     border-left: 1px solid #cc7b19;
 }
-.docker-setting {
+.setting-message {
     color: #cc7b19;
     margin-left: 10px;
 }
@@ -2183,7 +2184,7 @@ li.advanced-setting {
 }
 .user-info-username {
     font-size: 24px;
-    color: #fff;
+    color: #eee;
     padding-top: 27px;
     padding-left: 105px;
 }
@@ -2249,7 +2250,7 @@ li.advanced-setting {
     left: 0px;
 }
 .user-overview-stats-instance h3 strong{
-    color: #fff;
+    color: #eee;
 }
 .user-overview-stats-instance h3 {
     font-size: 30px;
@@ -2262,7 +2263,7 @@ li.advanced-setting {
     float: left;
 }
 .user-overview-stats-instance h4 {
-    color: #fff;
+    color: #eee;
     margin-bottom: 25px;
 }
 .user-overview-stats-instance h1 {
@@ -2302,7 +2303,7 @@ li.advanced-setting {
 .user-player-instance-name {
     float: left;
     padding-top: 14px;
-    color: #fff;
+    color: #eee;
     text-overflow: ellipsis;
     overflow: hidden;
     position: relative;
@@ -2312,6 +2313,7 @@ li.advanced-setting {
     width: 140px;
     margin-left: 10px;
     margin-bottom: 10px;
+    white-space: nowrap;
 }
 .user-player-instance-playcount h3 {
     font-size: 30px;
@@ -2360,9 +2362,6 @@ a .library-user-instance-box:hover {
     -moz-box-shadow: inset 0 0 0 2px #e9a049;
     box-shadow: inset 0 0 0 2px #e9a049;
 }
-#watched-stats-days-selection label {
-    margin-bottom: 0;
-}
 .home-padded-header {
     margin: 25px 0;
     height: 34px;
@@ -2440,7 +2439,7 @@ a .library-user-instance-box:hover {
     overflow: hidden;
 }
 .home-platforms-instance-name {
-    color: #fff;
+    color: #eee;
     text-overflow: ellipsis;
     overflow: hidden;
     position: relative;
@@ -2627,7 +2626,7 @@ a .library-user-instance-box:hover {
 }
 .home-platforms-instance-list-name {
     float: left;
-    color: #fff;
+    color: #eee;
     text-overflow: ellipsis;
     overflow: hidden;
     position: relative;
@@ -2994,6 +2993,9 @@ a .home-platforms-list-cover-face:hover
 .accordion li .link i.fa {
     color: #999;
 }
+.accordion li .link span.toggle-left {
+    padding-right: 5px;
+}
 .accordion li .link span.toggle-right {
     float: right;
     padding-left: 10px;
@@ -3039,7 +3041,7 @@ a .home-platforms-list-cover-face:hover
 }
 .submenu a:hover {
     background: #f9be03;
-    color: #FFF;
+    color: #eee;
 }
 .ajaxMsg {
     background-color: rgba(255,255,255,0.075);
@@ -3098,21 +3100,21 @@ div.dataTables_info {
     white-space: normal !important;
 }
 .tooltip.top .tooltip-arrow {
-    border-top-color: #fff;
+    border-top-color: #eee;
 }
 .tooltip.right .tooltip-arrow {
-    border-right-color: #fff;
+    border-right-color: #eee;
 }
 .tooltip.bottom .tooltip-arrow {
-    border-bottom-color: #fff;
+    border-bottom-color: #eee;
 }
 .tooltip.left .tooltip-arrow {
-    border-left-color: #fff;
+    border-left-color: #eee;
 }
 .tooltip-inner {
     max-width: 250px;
     color: #000;
-    background: #fff;
+    background: #eee;
     border: 0;
     font-weight: bold;
     border-radius: 2px;
@@ -3204,7 +3206,7 @@ div.dataTables_info {
 }
 .edit-user-toggles > input[type='checkbox']:checked + label,
 .edit-library-toggles > input[type='checkbox']:checked + label {
-    color: #fff;
+    color: #eee;
     cursor: pointer;
 }
 .edit-user-name > input[type='text'] {
@@ -3450,10 +3452,6 @@ pre::-webkit-scrollbar-thumb {
 .activity-queue tr:nth-child(even) td {
     background-color: rgba(255,255,255,0.010);
 }
-#days-selection label,
-#months-selection label {
-    margin-bottom: 0;
-}
 .card-sortable {
     height: 36px;
     padding: 0 20px 0 0;
@@ -3508,13 +3506,13 @@ pre::-webkit-scrollbar-thumb {
     width: 225px;
 }
 .config-scheduler-table th {
-    color: #fff;
+    color: #eee;
 }
 a.no-highlight {
     color: #777;
 }
 a.no-highlight:hover {
-    color: #fff;
+    color: #eee;
 }
 .top-line {
     border-top: 1px dotted #777;
@@ -3522,7 +3520,7 @@ a.no-highlight:hover {
 }
 .help-bold {
     font-weight: bold;
-    color: #fff;
+    color: #eee;
 }
 .save-button {
     margin-top: 15px;
@@ -3667,7 +3665,7 @@ a.no-highlight:hover {
     margin: 0 2px;
     padding: 2px 5px;
     font-size: 13px;
-    color: #fff;
+    color: #eee;
     background-color: #555;
     border: 0px solid #444;
     border-radius: 3px;
@@ -3685,7 +3683,7 @@ a.no-highlight:hover {
     -webkit-transition: all .1s cubic-bezier(.4,0,1,1);
     -moz-transition: all .1s cubic-bezier(.4,0,1,1);
     -o-transition: all .1s cubic-bezier(.4,0,1,1);
-    text-shadow: -1px -1px 0 #fff, 1px -1px 0 #fff, -1px 1px 0 #fff, 1px 1px 0 #fff;
+    text-shadow: -1px -1px 0 #eee, 1px -1px 0 #eee, -1px 1px 0 #eee, 1px 1px 0 #eee;
 }
 .overlay-refresh-image.left {
     left: 10px;
@@ -3699,7 +3697,7 @@ a.no-highlight:hover {
     cursor: pointer;
 }
 .overlay-refresh-image.info-art:hover {
-    color: #fff;
+    color: #eee;
     text-shadow: none;
 }
 a:hover .overlay-refresh-image {
@@ -3716,10 +3714,6 @@ a:hover .overlay-refresh-image:hover {
     padding-top: 10px;
     padding-bottom: 10px;
 }
-#plexpy-log-levels label,
-#plex-log-levels label {
-    margin-bottom: 0;
-}
 #plexpy-notifiers-table .friendly_name,
 #notifier-config-modal span.notifier_id,
 #plexpy-newsletters-table .friendly_name,
@@ -3754,7 +3748,7 @@ a:hover .overlay-refresh-image:hover {
 #newsletter-config-modal .nav-tabs > li.active > a,
 #newsletter-config-modal .nav-tabs > li.active > a:hover,
 #newsletter-config-modal .nav-tabs > li.active > a:focus {
-    color: #fff;
+    color: #eee;
     background: #222;
 }
 #notifier-config-modal .nav-tabs > li.active > a,
@@ -3870,6 +3864,10 @@ a:hover .overlay-refresh-image:hover {
     background-color: #31afe1;
     background-image: url(../images/platforms/kodi.svg);
 }
+.platform-lg {
+    background-color: #a50034;
+    background-image: url(../images/platforms/lg.svg);
+}
 .platform-linux {
     background-color: #1793d0;
     background-image: url(../images/platforms/linux.svg);
@@ -3971,6 +3969,9 @@ a:hover .overlay-refresh-image:hover {
 .platform-kodi-rgba {
     background-color: rgba(49, 175, 225, 0.40);
 }
+.platform-lg-rgba {
+    background-color: rgba(165, 0, 52, 0.40);
+}
 .platform-linux-rgba {
     background-color: rgba(23, 147, 208, 0.40);
 }
@@ -4058,6 +4059,11 @@ a:hover .overlay-refresh-image:hover {
     width: 62px !important;
     background-image: url(../images/rating/imdb.svg);
 }
+.rating-themoviedb {
+    width: 72px !important;
+    background-image: url(../images/rating/themoviedb.svg);
+    background-size: auto 16px !important;
+}
 .rating-rottentomatos-ripe {
     background-image: url(../images/rating/tomato-ripe.svg);
 }
@@ -4101,7 +4107,7 @@ a:hover .overlay-refresh-image:hover {
     flex-shrink: 0;
 }
 #info-modal .stream-info-item .sub-value {
-    color: #fff;
+    color: #eee;
     font-weight: bold;
     margin-left: 10px;
     text-align: left;
@@ -4124,7 +4130,7 @@ a:hover .overlay-refresh-image:hover {
 .stream-info th:first-child {
     width: 125px;
     height: 30px;
-    color: #fff;
+    color: #eee;
     font-size: 12px;
     text-align: right;
     text-transform: uppercase;
@@ -4247,7 +4253,7 @@ a[data-tab-destination] {
     transform: translate(-50%, -50%);
 }
 .iframe-button {
-    color: #fff;
+    color: #eee;
     border-radius: 20px;
     text-align: center;
     cursor: pointer;
@@ -4264,7 +4270,7 @@ a[data-tab-destination] {
 }
 .iframe-button:hover,
 .iframe-button:focus {
-    color: #fff;
+    color: #eee;
     box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 0px 99999px inset, rgba(0, 0, 0, 0.2) 0px 1px 5px 0px, rgba(0, 0, 0, 0.14) 0px 2px 2px 0px, rgba(0, 0, 0, 0.12) 0px 3px 1px -2px;
 }
 .iframe-button:active {
@@ -4297,3 +4303,40 @@ a[data-tab-destination] {
     margin-top: 0;
     color: #737373;
 }
+
+#browse-path-list > li > span > i.fa {
+    color: #999;
+}
+
+#tautulli-news .open .news-title,
+#tautulli-news .open .news-date,
+#tautulli-news .accordion li.open .link i.fa {
+    color: #eee;
+}
+.news-title,
+.news-date {
+    color: #999;
+    padding-left: 5px;
+}
+.news-subtitle {
+    display: block;
+    color: #aaa;
+    font-weight: bold;
+    margin-bottom: 10px;
+}
+.news-body {
+    display: block;
+    color: #aaa;
+}
+.news-body p:last-of-type {
+    margin-bottom: 0;
+}
+.news-body a {
+    display: inline !important;
+    background: none !important;
+    padding: 0 !important;
+    color: #eee;
+}
+.news-body a:hover {
+    color: #f9be03;
+}

data/interfaces/default/current_activity_instance.html

@@ -118,9 +118,7 @@ DOCUMENTATION :: END
                         <div id="poster-${sk}" class="dashboard-activity-cover" style="background-image: url(${page('pms_image_proxy', data['parent_thumb'], data['parent_rating_key'], 300, 300, fallback='cover', refresh=True)});"></div>
                     </a>
                     % elif data['media_type'] in ('photo', 'clip'):
-                        % if data['extra_type']:
+                        % if data['parent_thumb']:
-                        <div id="poster-${sk}" class="dashboard-activity-poster" style="background-image: url(${page('pms_image_proxy', data['art'].replace('/art', '/thumb') or data['thumb'], data['rating_key'], 300, 450, fallback='poster', refresh=True)});"></div>
-                        % elif data['parent_thumb']:
                         <div id="poster-${sk}" class="dashboard-activity-poster" style="background-image: url(${page('pms_image_proxy', data['parent_thumb'], data['parent_rating_key'], 300, 450, fallback='poster', refresh=True)});"></div>
                         % else:
                         <div id="poster-${sk}" class="dashboard-activity-poster" style="background-image: url(${page('pms_image_proxy', data['thumb'], data['rating_key'], 300, 450, fallback='poster', refresh=True)});"></div>
@@ -220,7 +218,7 @@ DOCUMENTATION :: END
                                 <div class="sub-heading">Container</div>
                                 <div class="sub-value" id="transcode_container-${sk}">
                                     % if data['stream_container_decision'] == 'transcode':
-                                    Transcode (${data['container'].upper()} <i class="fa fa-long-arrow-right"></i> ${data['stream_container'].upper()})
+                                    Converting (${data['container'].upper()} <i class="fa fa-long-arrow-right"></i> ${data['stream_container'].upper()})
                                     % else:
                                     Direct Play (${data['stream_container'].upper()})
                                     % endif
@@ -399,7 +397,7 @@ DOCUMENTATION :: END
         </div>
     </div>
     <div class="dashboard-activity-metadata-wrapper">
-        <a href="${user_href}" title="${data['friendly_name']}">
+        <a href="${user_href}" title="${data['username']}">
             <div class="dashboard-activity-metadata-user-thumb" style="background-image: url(${data['user_thumb']});"></div>
         </a>
         <div class="dashboard-activity-metadata-title-container">
@@ -410,6 +408,8 @@ DOCUMENTATION :: END
                 <i class="fa fa-fw fa-pause"></i>&nbsp;
                 % elif data['state'] == 'buffering':
                 <i class="fa fa-fw fa-spinner"></i>&nbsp;
+                % elif data['state'] == 'error':
+                <i class="fa fa-fw fa-exclamation-triangle"></i>&nbsp;
                 % endif
             </div>
             <div class="dashboard-activity-metadata-title">
@@ -521,7 +521,7 @@ DOCUMENTATION :: END
                 % endif
             </div>
             <div class="dashboard-activity-metadata-user">
-                <a href="${user_href}" title="${data['friendly_name']}">${data['friendly_name']}</a>
+                <a href="${user_href}" title="${data['username']}">${data['friendly_name']}</a>
             </div>
         </div>
     </div>

data/interfaces/default/edit_library.html

@@ -115,21 +115,13 @@ DOCUMENTATION :: END
         var msg = 'Are you REALLY sure you want to purge all history for the <strong>${data["section_name"]}</strong> library?<br>' + 
             'This is permanent and cannot be undone!';
         var url = 'delete_all_library_history';
-        confirmAjaxCall(url, msg, { section_id: '${data["section_id"]}' }, null, function () { location.reload(); });
+        confirmAjaxCall(url, msg, { server_id: '${server_id}', section_id: '${data["section_id"]}' }, null, function () { location.reload(); });
     });
 
     $('#undelete-library').click(function () {
-        var msg = 'Are you sure you want to undelete this user?';
+        var msg = 'Are you sure you want to undelete this library?';
         var url = 'undelete_library';
         confirmAjaxCall(url, msg, { section_id: '${data["section_id"]}' }, null, function () { location.reload(); });
     });
-
-    $(document).ready(function() {
-        // Move #confirm-modal to parent container
-        if (!($('#edit-library-modal').next().is('#confirm-modal-purge'))) {
-            $('#confirm-modal-purge').appendTo($('#edit-library-modal').parent());
-        }
-        $('#edit-library-modal > #confirm-modal-purge').remove();
-    });
 </script>
 % endif

data/interfaces/default/edit_user.html

@@ -134,13 +134,5 @@ DOCUMENTATION :: END
         var url = 'undelete_user';
         confirmAjaxCall(url, msg, { user_id: '${data["user_id"]}' }, null, function () { location.reload(); });
     });
-
-    $(document).ready(function() {
-        // Move #confirm-modal-purge to parent container
-        if (!($('#edit-user-modal').next().is('#confirm-modal-purge'))) {
-            $('#confirm-modal-purge').appendTo($('#edit-user-modal').parent());
-        }
-        $('#edit-user-modal > #confirm-modal-purge').remove();
-    });
 </script>
 % endif

data/interfaces/default/history.html

@@ -8,6 +8,13 @@
 
 <%def name="body()">
 <div class='container-fluid'>
+    % if config['database_is_importing']:
+    <div style="text-align: center; margin-top: 20px;">
+        <i class="fa fa-refresh fa-spin"></i>&nbsp; Tautulli is importing history from another database. This could take a few minutes depending on the size of your database.
+        <br />
+        You may leave this page and check back later.
+    </div>
+    % endif
     <div class='table-card-header'>
         <div class="header-bar">
             <span><i class="fa fa-history"></i> History</span>

data/interfaces/default/home_stats.html

@@ -104,7 +104,7 @@ DOCUMENTATION :: END
             </div>
             % elif stat_id == 'top_users':
             <% user_href = page('user', row0['user_id']) if row0['user_id'] else '#' %>
-            <a id="stats-thumb-url-${stat_id}" href="${user_href}" title="${row0['friendly_name']}" class="hidden-xs">
+            <a id="stats-thumb-url-${stat_id}" href="${user_href}" title="${row0['user']}" class="hidden-xs">
                 <div id="stats-thumb-${stat_id}" class="dashboard-stats-circle" style="background-image: url(${row0['user_thumb'] or 'images/gravatar-default.png'})"></div>
             </a>
             % elif stat_id == 'top_platforms':
@@ -122,7 +122,7 @@ DOCUMENTATION :: END
                     % elif stat_id.startswith('popular'):
                     <span class="dashboard-stats-stats-units">users</span>
                     % elif stat_id == 'last_watched':
-                    <span class="dashboard-stats-stats-units" id="last-watched-header-info">${row0['friendly_name']}</span>
+                    <span class="dashboard-stats-stats-units" id="last-watched-header-info" title="${row0['user']}">${row0['friendly_name']}</span>
                     % elif stat_id == 'most_concurrent':
                     <span class="dashboard-stats-stats-units" id="most-concurrent-header-info">streams</span>
                     % endif
@@ -134,7 +134,7 @@ DOCUMENTATION :: END
                             <li class="dashboard-stats-info-item ${'expanded' if loop.index == 0 else ''}" data-stat_id="${stat_id}"
                                 data-rating_key="${row.get('rating_key')}" data-guid="${row.get('guid')}" data-title="${row.get('title')}"
                                 data-art="${row.get('art')}" data-thumb="${row.get('thumb')}" data-platform="${row.get('platform_name')}"
-                                data-user_id="${row.get('user_id')}" data-friendly_name="${row.get('friendly_name')}" data-user_thumb="${row.get('user_thumb')}"
+                                data-user_id="${row.get('user_id')}" data-user="${row.get('user')}" data-friendly_name="${row.get('friendly_name')}" data-user_thumb="${row.get('user_thumb')}"
                                 data-last_watch="${row.get('last_watch')}" data-started="${row.get('started')}" data-live="${row.get('live')}">
                                 <div class="sub-list">${loop.index + 1}</div>
                                 <div class="sub-value">
@@ -152,7 +152,7 @@ DOCUMENTATION :: END
                                     </a>
                                     % elif stat_id == 'top_users':
                                     <% user_href = page('user', row['user_id']) if row['user_id'] else '#' %>
-                                    <a href="${user_href}" title="${row['friendly_name']}">
+                                    <a href="${user_href}" title="${row['user']}">
                                         ${row['friendly_name']}
                                     </a>
                                     % elif stat_id == 'top_platforms':

data/interfaces/default/images/platforms/lg.svg

@@ -0,0 +1,7 @@
+<!-- Generated by IcoMoon.io -->
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="64" height="64" viewBox="0 0 64 64">
+<title>lg</title>
+<path fill="#fff" d="M30.203 31.797c0 8.176-6.654 14.832-14.835 14.82-7.927-0.011-14.818-6.28-14.812-14.838 0.005-8.282 6.541-14.82 14.841-14.803 8.618 0.017 14.807 6.969 14.806 14.822zM26.577 32.388c-0.087 4.433-3.485 9.518-9.37 10.487-6.122 1.008-11.584-2.989-12.814-8.656-0.632-2.912-0.221-5.696 1.362-8.228 2.347-3.754 5.815-5.502 10.222-5.453 0-0.387 0-0.761 0-1.134-4.114-0.281-9.226 1.824-11.763 6.923-2.454 4.932-1.296 10.953 2.811 14.672 4.153 3.762 10.224 4.309 14.953 1.326 2.328-1.468 3.999-3.496 4.997-6.067 0.628-1.617 0.882-3.296 0.813-5.032-2.967 0-5.909 0-8.864 0 0 0.39 0 0.768 0 1.162 2.558-0 5.097-0 7.652-0zM15.991 37.112c0-0.129 0-0.221 0-0.313 0-3.731 0-7.463 0-11.194 0-0.060-0.004-0.119 0-0.179 0.009-0.118-0.038-0.166-0.16-0.163-0.278 0.006-0.556 0.012-0.833-0.002-0.178-0.008-0.237 0.042-0.237 0.23 0.005 4.194 0.005 8.389-0 12.583-0 0.198 0.065 0.239 0.249 0.237 1.224-0.007 2.448-0.004 3.672-0.004 0.072 0 0.143 0 0.244 0 0-0.343-0.008-0.665 0.003-0.987 0.006-0.166-0.050-0.214-0.214-0.212-0.82 0.007-1.641 0.003-2.461 0.003-0.078 0-0.155 0-0.263 0zM12.434 27.068c0.003-0.987-0.799-1.798-1.785-1.805s-1.799 0.796-1.805 1.782c-0.006 0.985 0.799 1.8 1.783 1.805 0.985 0.004 1.804-0.803 1.807-1.783z"></path>
+<path fill="#fff" d="M63.467 30.606c0 2.864 0 5.707 0 8.571-1.242 0-2.479 0-3.742 0 0-0.468 0-0.933 0-1.433-0.203 0.226-0.366 0.432-0.553 0.612-0.683 0.656-1.518 1-2.441 1.136-1.187 0.174-2.348 0.075-3.462-0.4-1.234-0.526-2.145-1.407-2.8-2.565-0.599-1.058-0.906-2.207-1.035-3.409-0.148-1.367-0.103-2.723 0.28-4.051 0.797-2.764 2.635-4.391 5.453-4.899 1.534-0.277 3.058-0.208 4.54 0.311 1.243 0.436 2.298 1.139 3.011 2.276 0.431 0.688 0.584 1.467 0.687 2.258 0.013 0.097 0.028 0.195 0.046 0.318-0.064 0.003-0.126 0.010-0.188 0.010-1.389 0.001-2.779-0.002-4.169 0.003-0.151 0.001-0.215-0.034-0.245-0.197-0.229-1.234-1.281-1.773-2.308-1.679-1.182 0.108-1.823 0.859-2.22 1.888-0.211 0.547-0.315 1.12-0.352 1.703-0.066 1.061-0.039 2.117 0.31 3.138 0.211 0.618 0.523 1.173 1.050 1.579 1.371 1.055 3.326 0.436 3.877-1.228 0.090-0.274 0.157-0.557 0.246-0.875-0.112 0-0.182 0-0.251 0-0.794 0-1.588-0.005-2.382 0.004-0.168 0.002-0.213-0.053-0.212-0.214 0.006-0.887 0.005-1.774 0.001-2.66-0.001-0.139 0.019-0.215 0.192-0.215 2.17 0.006 4.341 0.004 6.511 0.004 0.045 0 0.090 0.008 0.157 0.013z"></path>
+<path fill="#fff" d="M48.501 35.522c0 1.233 0 2.44 0 3.661-3.613 0-7.216 0-10.834 0 0-4.923 0-9.841 0-14.77 1.44 0 2.872 0 4.331 0 0 0.092 0 0.175 0 0.259 0 3.526 0 7.053 0 10.579 0 0.271 0 0.271 0.267 0.271 1.985 0 3.97 0 5.954 0 0.086 0 0.171 0 0.281 0z"></path>
+</svg>

data/interfaces/default/images/rating/themoviedb.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 190.24 81.52"><defs><linearGradient id="a" y1="40.76" x2="190.24" y2="40.76" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#90cea1"/><stop offset=".56" stop-color="#3cbec9"/><stop offset="1" stop-color="#00b3e5"/></linearGradient></defs><g data-name="Layer 2"><path d="M105.67 36.06h66.9a17.67 17.67 0 0017.67-17.66A17.67 17.67 0 00172.57.73h-66.9A17.67 17.67 0 0088 18.4a17.67 17.67 0 0017.67 17.66zm-88 45h76.9a17.67 17.67 0 0017.67-17.66 17.67 17.67 0 00-17.67-17.67h-76.9A17.67 17.67 0 000 63.4a17.67 17.67 0 0017.67 17.66zm-7.26-45.64h7.8V6.92h10.1V0h-28v6.9h10.1zm28.1 0h7.8V8.25h.1l9 27.15h6l9.3-27.15h.1V35.4h7.8V0H66.76l-8.2 23.1h-.1L50.31 0h-11.8zm113.92 20.25a15.07 15.07 0 00-4.52-5.52 18.57 18.57 0 00-6.68-3.08 33.54 33.54 0 00-8.07-1h-11.7v35.4h12.75a24.58 24.58 0 007.55-1.15 19.34 19.34 0 006.35-3.32 16.27 16.27 0 004.37-5.5 16.91 16.91 0 001.63-7.58 18.5 18.5 0 00-1.68-8.25zM145 68.6a8.8 8.8 0 01-2.64 3.4 10.7 10.7 0 01-4 1.82 21.57 21.57 0 01-5 .55h-4.05v-21h4.6a17 17 0 014.67.63 11.66 11.66 0 013.88 1.87A9.14 9.14 0 01145 59a9.87 9.87 0 011 4.52 11.89 11.89 0 01-1 5.08zm44.63-.13a8 8 0 00-1.58-2.62 8.38 8.38 0 00-2.42-1.85 10.31 10.31 0 00-3.17-1v-.1a9.22 9.22 0 004.42-2.82 7.43 7.43 0 001.68-5 8.42 8.42 0 00-1.15-4.65 8.09 8.09 0 00-3-2.72 12.56 12.56 0 00-4.18-1.3 32.84 32.84 0 00-4.62-.33h-13.2v35.4h14.5a22.41 22.41 0 004.72-.5 13.53 13.53 0 004.28-1.65 9.42 9.42 0 003.1-3 8.52 8.52 0 001.2-4.68 9.39 9.39 0 00-.55-3.18zm-19.42-15.75h5.3a10 10 0 011.85.18 6.18 6.18 0 011.7.57 3.39 3.39 0 011.22 1.13 3.22 3.22 0 01.48 1.82 3.63 3.63 0 01-.43 1.8 3.4 3.4 0 01-1.12 1.2 4.92 4.92 0 01-1.58.65 7.51 7.51 0 01-1.77.2h-5.65zm11.72 20a3.9 3.9 0 01-1.22 1.3 4.64 4.64 0 01-1.68.7 8.18 8.18 0 01-1.82.2h-7v-8h5.9a15.35 15.35 0 012 .15 8.47 8.47 0 012.05.55 4 4 0 011.57 1.18 3.11 3.11 0 01.63 2 3.71 3.71 0 01-.43 1.92z" fill="url(#a)" data-name="Layer 1"/></g></svg>

data/interfaces/default/index.html

@@ -179,10 +179,10 @@
         <div class="modal-content">
             <div class="modal-header">
                 <button type="button" class="close" data-dismiss="modal" aria-hidden="true"><i class="fa fa-remove"></i></button>
-                <h4 class="modal-title">Terminate Session</h4>
+                <h4 class="modal-title">Terminate Stream</h4>
             </div>
             <div class="modal-body" style="text-align: center;">
-                <p>Are you sure you want to terminate this session?</p>
+                <p>Are you sure you want to terminate this stream?</p>
                 <p>
                     <strong>
                         <span id="terminate-user"></span><br />
@@ -377,6 +377,9 @@
                             case 'buffering':
                                 state_icon = '<i class="fa fa-fw fa-spinner"></i>&nbsp;';
                                 break;
+                            case 'error':
+                                state_icon = '<i class="fa fa-fw fa-exclamation-triangle"></i>&nbsp;';
+                                break;
                             default:
                                 state_icon = '<i class="fa fa-fw fa-question-circle"></i>&nbsp;';
                         }
@@ -431,7 +434,7 @@
 
                         var transcode_container = '';
                         if (s.stream_container_decision === 'transcode') {
-                            transcode_container = 'Transcode (' + s.container.toUpperCase() + ' <i class="fa fa-long-arrow-right"></i> ' + s.stream_container.toUpperCase() + ')';
+                            transcode_container = 'Converting (' + s.container.toUpperCase() + ' <i class="fa fa-long-arrow-right"></i> ' + s.stream_container.toUpperCase() + ')';
                         } else {
                             transcode_container = 'Direct Play (' + s.stream_container.toUpperCase() + ')';
                         }
@@ -756,7 +759,7 @@
                 if (user_id) {
                     href = page('user', user_id);
                 }
-                $('#stats-thumb-url-' + stat_id).attr('href', href).prop('title', $(elem).data('friendly_name'));
+                $('#stats-thumb-url-' + stat_id).attr('href', href).prop('title', $(elem).data('user'));
             } else if (stat_id === 'top_platforms') {
                 $('#stats-thumb-' + stat_id).removeClass(function (index, className) {
                     return (className.match (/(^|\s)platform-\S+/g) || []).join(' ');

data/interfaces/default/info.html

@@ -45,14 +45,14 @@ DOCUMENTATION :: END
 
     # Get audio codec file
     def af(codec):
-        for pattern, file_type in MEDIA_FLAGS_AUDIO.iteritems():
+        for pattern, file_type in MEDIA_FLAGS_AUDIO.items():
             if re.match(pattern, codec):
                 return file_type
         return codec
 
     # Get video codec file
     def vf(codec):
-        for pattern, file_type in MEDIA_FLAGS_VIDEO.iteritems():
+        for pattern, file_type in MEDIA_FLAGS_VIDEO.items():
             if re.match(pattern, codec):
                 return file_type
         return codec
@@ -275,6 +275,11 @@ DOCUMENTATION :: END
                                 <span class="rating-image rating-imdb"><strong>${data['rating']}</strong></span>
                             </div>
                             % endif
+                            % if data['rating_image'].startswith('themoviedb://'):
+                            <div class="critic-rating hidden-xs hidden-sm" title="${data['rating']}">
+                                <span class="rating-image rating-themoviedb"><strong>${get_percent(data['rating'], 10)}%</strong></span>
+                            </div>
+                            % endif
                             % if data['audience_rating_image'].startswith('rottentomatoes://'):
                             <div class="critic-rating hidden-xs hidden-sm" title="${data['audience_rating']}">
                                 <span class="rating-image rating-rottentomatos-${data['audience_rating_image'].rsplit('.')[-1]}"><strong>${get_percent(data['audience_rating'], 10)}%</strong></span>

data/interfaces/default/info_search_results_list.html

@@ -65,7 +65,7 @@ DOCUMENTATION :: END
     <ul class="item-children-instance list-unstyled">
         % for child in data['results_list']['collection']:
         <li>
-            <a href="${page('info', child['rating_key'])}" id="${child['rating_key']}">
+            <a href="${page('info', child['rating_key'])}" data-rating_key="${child['rating_key']}" data-library_name="${child['library_name']}">
                 <div class="item-children-poster">
                     <div class="item-children-poster-face poster-item" style="background-image: url(${page('pms_image_proxy', child['thumb'], child['rating_key'], 300, 450, fallback='poster')});"></div>
                     % if _session['user_group'] == 'admin':
@@ -90,7 +90,7 @@ DOCUMENTATION :: END
     <ul class="item-children-instance list-unstyled">
         % for child in data['results_list']['movie']:
         <li>
-            <a href="${page('info', child['rating_key'])}" id="${child['rating_key']}">
+            <a href="${page('info', child['rating_key'])}" data-rating_key="${child['rating_key']}" data-library_name="${child['library_name']}">
                 <div class="item-children-poster">
                     <div class="item-children-poster-face poster-item" style="background-image: url(${page('pms_image_proxy', child['thumb'], child['rating_key'], 300, 450, fallback='poster')});"></div>
                     % if _session['user_group'] == 'admin':
@@ -115,7 +115,7 @@ DOCUMENTATION :: END
     <ul class="item-children-instance list-unstyled">
         % for child in data['results_list']['show']:
         <li>
-            <a href="${page('info', child['rating_key'])}" id="${child['rating_key']}">
+            <a href="${page('info', child['rating_key'])}" data-rating_key="${child['rating_key']}" data-library_name="${child['library_name']}">
                 <div class="item-children-poster">
                     <div class="item-children-poster-face poster-item" style="background-image: url(${page('pms_image_proxy', child['thumb'], child['rating_key'], 300, 450, fallback='poster')});"></div>
                     % if _session['user_group'] == 'admin':
@@ -140,7 +140,7 @@ DOCUMENTATION :: END
     <ul class="item-children-instance list-unstyled">
         % for child in data['results_list']['season']:
         <li>
-            <a href="${page('info', child['rating_key'])}" id="${child['rating_key']}">
+            <a href="${page('info', child['rating_key'])}" data-rating_key="${child['rating_key']}" data-library_name="${child['library_name']}">
                 <div class="item-children-poster">
                     <div class="item-children-poster-face poster-item" style="background-image: url(${page('pms_image_proxy', child['thumb'], child['rating_key'], 300, 450, fallback='poster')});"></div>
                     % if _session['user_group'] == 'admin':
@@ -165,7 +165,7 @@ DOCUMENTATION :: END
     <ul class="item-children-instance list-unstyled">
         % for child in data['results_list']['episode']:
         <li>
-            <a href="${page('info', child['rating_key'])}" id="${child['rating_key']}">
+            <a href="${page('info', child['rating_key'])}" data-rating_key="${child['rating_key']}" data-library_name="${child['library_name']}">
                 <div class="item-children-poster">
                     <div class="item-children-poster-face episode-item" style="background-image: url(${page('pms_image_proxy', child['thumb'], child['rating_key'], 500, 280, fallback='art')});"></div>
                     % if _session['user_group'] == 'admin':
@@ -191,7 +191,7 @@ DOCUMENTATION :: END
     <ul class="item-children-instance list-unstyled">
         % for child in data['results_list']['artist']:
         <li>
-            <a href="${page('info', child['rating_key'])}" id="${child['rating_key']}">
+            <a href="${page('info', child['rating_key'])}" data-rating_key="${child['rating_key']}" data-library_name="${child['library_name']}">
                 <div class="item-children-poster">
                     <div class="item-children-poster-face cover-item" style="background-image: url(${page('pms_image_proxy', child['thumb'], child['rating_key'], 300, 300, fallback='cover')});"></div>
                     % if _session['user_group'] == 'admin':
@@ -215,7 +215,7 @@ DOCUMENTATION :: END
     <ul class="item-children-instance list-unstyled">
         % for child in data['results_list']['album']:
         <li>
-            <a href="${page('info', child['rating_key'])}" id="${child['rating_key']}">
+            <a href="${page('info', child['rating_key'])}" data-rating_key="${child['rating_key']}" data-library_name="${child['library_name']}">
                 <div class="item-children-poster">
                     <div class="item-children-poster-face cover-item" style="background-image: url(${page('pms_image_proxy', child['thumb'], child['rating_key'], 300, 300, fallback='cover')});"></div>
                     % if _session['user_group'] == 'admin':
@@ -240,7 +240,7 @@ DOCUMENTATION :: END
     <ul class="item-children-instance list-unstyled">
         % for child in data['results_list']['track']:
         <li>
-            <a href="${page('info', child['rating_key'])}" id="${child['rating_key']}">
+            <a href="${page('info', child['rating_key'])}" data-rating_key="${child['rating_key']}" data-library_name="${child['library_name']}">
                 <div class="item-children-poster">
                     <div class="item-children-poster-face cover-item" style="background-image: url(${page('pms_image_proxy', child['parent_thumb'], child['parent_rating_key'], 300, 300, fallback='cover')});">
                         <div class="item-children-card-overlay">

data/interfaces/default/js/ajaxNotifications.js

@@ -36,7 +36,3 @@ function check_notifications() {
         check_notifications();
     }, 5000);
 }
-
-$(document).ready(function () {
-    check_notifications();
-});

data/interfaces/default/js/script.js

@@ -237,6 +237,27 @@ function doAjaxCall(url, elem, reload, form, showMsg, callback) {
     });
 }
 
+getBrowsePath = function (key, path, filter_ext) {
+    var deferred = $.Deferred();
+
+    $.ajax({
+        url: 'browse_path',
+        type: 'GET',
+        data: {
+            key: key,
+            path: path,
+            filter_ext: filter_ext
+        },
+        success: function(data) {
+            deferred.resolve(data);
+        },
+        error: function() {
+            deferred.reject();
+        }
+    });
+    return deferred;
+};
+
 function doSimpleAjaxCall(url) {
     $.ajax(url);
 }

data/interfaces/default/js/tables/history_table.js

@@ -60,6 +60,8 @@ history_table_options = {
                         state = '<span class="current-activity-tooltip" data-toggle="tooltip" title="Currently Paused"><i class="fa fa-pause fa-fw"></i></span>';
                     } else if (rowData['state'] === 'buffering') {
                         state = '<span class="current-activity-tooltip" data-toggle="tooltip" title="Currently Buffering"><i class="fa fa-spinner fa-fw"></i></span>';
+                    } else if (rowData['state'] === 'error') {
+                        state = '<span class="current-activity-tooltip" data-toggle="tooltip" title="Playback Error"><i class="fa fa-exclamation-triangle fa-fw"></i></span>';
                     } else if (rowData['state'] === 'stopped') {
                         state = '<span class="current-activity-tooltip" data-toggle="tooltip" title="Currently Stopped"><i class="fa fa-stop fa-fw"></i></span>';
                     }
@@ -81,9 +83,9 @@ history_table_options = {
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData !== '') {
                     if (rowData['user_id']) {
-                        $(td).html('<a href="' + page('user', rowData['user_id']) + '">' + cellData + '</a>');
+                        $(td).html('<a href="' + page('user', rowData['user_id']) + '" title="' + rowData['user'] + '">' + cellData + '</a>');
                     } else {
-                        $(td).html('<a href="' + page('user', null, rowData['user']) + '">' + cellData + '</a>');
+                        $(td).html('<a href="' + page('user', null, rowData['user']) + '" title="' + rowData['user'] + '">' + cellData + '</a>');
                     }
                 } else {
                     $(td).html(cellData);
@@ -141,7 +143,7 @@ history_table_options = {
                     if (rowData['transcode_decision'] === 'transcode') {
                         transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Transcode"><i class="fa fa-server fa-fw"></i></span>';
                     } else if (rowData['transcode_decision'] === 'copy') {
-                        transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Stream"><i class="fa fa-video-camera fa-fw"></i></span>';
+                        transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Stream"><i class="fa fa-stream fa-fw"></i></span>';
                     } else if (rowData['transcode_decision'] === 'direct play') {
                         transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Play"><i class="fa fa-play-circle fa-fw"></i></span>';
                     }
@@ -184,7 +186,9 @@ history_table_options = {
                         thumb_popover = '<span class="thumb-tooltip" data-toggle="popover" data-img="' + page('pms_image_proxy', rowData['thumb'], rowData['rating_key'], 300, 300, null, null, null, 'cover') + '" data-height="80" data-width="80">' + cellData + parent_info + '</span>';
                         $(td).html('<div class="history-title"><a href="' + page('info', rowData['rating_key'], rowData['guid'], history, rowData['live']) + '"><div style="float: left;">' + media_type + '&nbsp;' + thumb_popover + '</div></a></div>');
                     } else if (rowData['media_type'] === 'clip') {
-                        $(td).html(cellData);
+                        media_type = '<span class="media-type-tooltip" data-toggle="tooltip" title="Clip"><i class="fa fa-video-camera fa-fw"></i></span>';
+                        thumb_popover = '<span class="thumb-tooltip" data-toggle="popover" data-img="' + page('pms_image_proxy', rowData['thumb'], rowData['rating_key'], 300, 450, null, null, null, fallback) + '" data-height="120" data-width="80">' + cellData + parent_info + '</span>';
+                        $(td).html('<div class="history-title"><div style="float: left;">' + media_type + '&nbsp;' + thumb_popover + '</div></div>');
                     } else {
                         $(td).html('<a href="' + page('info', rowData['rating_key']) + '">' + cellData + '</a>');
                     }

data/interfaces/default/js/tables/history_table_modal.js

@@ -83,7 +83,7 @@ history_table_modal_options = {
                     if (rowData['transcode_decision'] === 'transcode') {
                         transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Transcode"><i class="fa fa-server fa-fw"></i></span>';
                     } else if (rowData['transcode_decision'] === 'copy') {
-                        transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Stream"><i class="fa fa-video-camera fa-fw"></i></span>';
+                        transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Stream"><i class="fa fa-stream fa-fw"></i></span>';
                     } else if (rowData['transcode_decision'] === 'direct play') {
                         transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Play"><i class="fa fa-play-circle fa-fw"></i></span>';
                     }

data/interfaces/default/js/tables/sync_table.js

@@ -51,9 +51,9 @@ sync_table_options = {
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData !== '') {
                     if (rowData['user_id']) {
-                        $(td).html('<a href="' + page('user', rowData['user_id']) + '">' + cellData + '</a>');
+                        $(td).html('<a href="' + page('user', rowData['user_id']) + '" title="' + rowData['username'] + '">' + cellData + '</a>');
                     } else {
-                        $(td).html('<a href="' + page('user', null, rowData['user']) + '">' + cellData + '</a>');
+                        $(td).html('<a href="' + page('user', null, rowData['user']) + '" title="' + rowData['username'] + '">' + cellData + '</a>');
                     }
                 } else {
                     $(td).html(cellData);

data/interfaces/default/js/tables/user_ips.js

@@ -1,3 +1,25 @@
+var date_format = 'YYYY-MM-DD';
+var time_format = 'hh:mm a';
+
+$.ajax({
+  url: 'get_date_formats',
+  type: 'GET',
+  success: function(data) {
+      date_format = data.date_format;
+      time_format = data.time_format;
+  }
+});
+
+var seenRender = function (data, type, full) {
+  return moment(data, "X").fromNow();
+};
+
+var seenCreatedCell = function (td, cellData, rowData, row, col) {
+  if (cellData !== null) {
+    $(td).attr('title', moment(cellData, "X").format(date_format + ' ' + time_format));
+  }
+};
+
 user_ip_table_options = {
     "destroy": true,
     "language": {
@@ -22,15 +44,23 @@ user_ip_table_options = {
         {
             "targets": [0],
             "data": "last_seen",
-            "render": function ( data, type, full ) {
+            "render": seenRender,
-                return moment(data, "X").fromNow();
+            "createdCell": seenCreatedCell,
-            },
             "searchable": false,
-            "width": "15%",
+            "width": "12%",
             "className": "no-wrap"
         },
         {
             "targets": [1],
+            "data": "first_seen",
+            "render": seenRender,
+            "createdCell": seenCreatedCell,
+            "searchable": false,
+            "width": "12%",
+            "className": "no-wrap"
+        },
+        {
+            "targets": [2],
             "data": "ip_address",
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData) {
@@ -44,22 +74,22 @@ user_ip_table_options = {
                     $(td).html('n/a');
                 }
             },
-            "width": "15%",
+            "width": "12%",
             "className": "no-wrap modal-control-ip"
         },
         {
-            "targets": [2],
+            "targets": [3],
             "data": "platform",
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData !== '') {
                     $(td).html(cellData);
                 }
             },
-            "width": "15%",
+            "width": "12%",
             "className": "no-wrap"
         },
         {
-            "targets": [3],
+            "targets": [4],
             "data": "player",
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData !== '') {
@@ -67,18 +97,18 @@ user_ip_table_options = {
                     if (rowData['transcode_decision'] === 'transcode') {
                         transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Transcode"><i class="fa fa-server fa-fw"></i></span>';
                     } else if (rowData['transcode_decision'] === 'copy') {
-                        transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Stream"><i class="fa fa-video-camera fa-fw"></i></span>';
+                        transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Stream"><i class="fa fa-stream fa-fw"></i></span>';
                     } else if (rowData['transcode_decision'] === 'direct play') {
                         transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Play"><i class="fa fa-play-circle fa-fw"></i></span>';
                     }
                     $(td).html('<div><a href="#" data-target="#info-modal" data-toggle="modal"><div style="float: left;">' + transcode_dec + '&nbsp;' + cellData + '</div></a></div>');
                 }
             },
-            "width": "15%",
+            "width": "12%",
             "className": "no-wrap modal-control"
         },
         {
-            "targets": [4],
+            "targets": [5],
             "data": "last_played",
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData !== '') {
@@ -119,7 +149,7 @@ user_ip_table_options = {
             "className": "datatable-wrap"
         },
         {
-            "targets": [5],
+            "targets": [6],
             "data": "play_count",
             "searchable": false,
             "width": "10%",

data/interfaces/default/js/tables/users.js

@@ -62,9 +62,9 @@ users_list_table_options = {
                 var inactive = '';
                 if (!rowData['is_active']) { inactive = '<span class="inactive-user-tooltip" data-toggle="tooltip" title="User not on Plex server"><i class="fa fa-exclamation-triangle"></i></span>'; }
                 if (cellData === '') {
-                    $(td).html('<a href="' + page('user', rowData['user_id']) + '"><div class="users-poster-face" style="background-image: url(../../images/gravatar-default-80x80.png);">' + inactive + '</div></a>');
+                    $(td).html('<a href="' + page('user', rowData['user_id']) + '"" title="' + rowData['username'] + '"><div class="users-poster-face" style="background-image: url(../../images/gravatar-default-80x80.png);">' + inactive + '</div></a>');
                 } else {
-                    $(td).html('<a href="' + page('user', rowData['user_id']) + '"><div class="users-poster-face" style="background-image: url(' + rowData['user_thumb'] + ');">' + inactive + '</div></a>');
+                    $(td).html('<a href="' + page('user', rowData['user_id']) + '"" title="' + rowData['username'] + '"><div class="users-poster-face" style="background-image: url(' + rowData['user_thumb'] + ');">' + inactive + '</div></a>');
                 }
             },
             "orderable": false,
@@ -78,7 +78,7 @@ users_list_table_options = {
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData !== null && cellData !== '') {
                     $(td).html('<div class="edit-user-name" data-id="' + rowData['row_id'] + '">' +
-                        '<a href="' + page('user', rowData['user_id']) + '">' + cellData + '</a>' +
+                        '<a href="' + page('user', rowData['user_id']) + '" title="' + rowData['username'] + '">' + cellData + '</a>' +
                         '<input type="text" class="hidden" value="' + cellData + '">' +
                         '</div>');
                 } else {
@@ -142,7 +142,7 @@ users_list_table_options = {
                     if (rowData['transcode_decision'] === 'transcode') {
                         transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Transcode"><i class="fa fa-server fa-fw"></i></span>';
                     } else if (rowData['transcode_decision'] === 'copy') {
-                        transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Stream"><i class="fa fa-video-camera fa-fw"></i></span>';
+                        transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Stream"><i class="fa fa-stream fa-fw"></i></span>';
                     } else if (rowData['transcode_decision'] === 'direct play') {
                         transcode_dec = '<span class="transcode-tooltip" data-toggle="tooltip" title="Direct Play"><i class="fa fa-play-circle fa-fw"></i></span>';
                     }

data/interfaces/default/libraries.html

@@ -38,7 +38,7 @@
                 <th align="left" id="count">Total Movies / TV Shows / Artists</th>
                 <th align="left" id="parent_count">Total Seasons / Albums</th>
                 <th align="left" id="child_count">Total Episodes / Tracks</th>
-                <th align="left" id="last_accessed">Last Accessed</th>
+                <th align="left" id="last_accessed">Last Streamed</th>
                 <th align="left" id="last_played">Last Played</th>
                 <th align="left" id="total_plays">Total Plays</th>
                 <th align="left" id="total_duration">Total Played Duration</th>

data/interfaces/default/library.html

@@ -251,9 +251,9 @@ DOCUMENTATION :: END
                                     % else:
                                     <div id="get_file_sizes_message" style="text-align: center; margin-top: 20px; display: none;">
                                     % endif
-                                        <i class="fa fa-refresh fa-spin"></i> Tautulli is calculating the file sizes for the library's media info. This could take a few minutes depending on the size of your library.
+                                        <i class="fa fa-refresh fa-spin"></i>&nbsp; Tautulli is calculating the file sizes for the library's media info. This could take a few minutes depending on the size of your library.
                                         <br />
-                                        You may leave this page and come back later.
+                                        You may leave this page and check back later.
                                     </div>
                                     <div class='table-card-header'>
                                         <div class="header-bar">

data/interfaces/default/library_user_stats.html

@@ -25,11 +25,11 @@ DOCUMENTATION :: END
     <div class="user-player-instance">
         <li>
             % if a['user_id']:
-            <a href="${page('user', a['user_id'])}" title="${a['friendly_name']}">
+            <a href="${page('user', a['user_id'])}" title="${a['username']}">
                 <div class="library-user-instance-box" style="background-image: url(${a['user_thumb']});"></div>
             </a>
             <div class=" user-player-instance-name">
-                <a href="${page('user', a['user_id'])}" title="${a['friendly_name']}">${a['friendly_name']}</a>
+                <a href="${page('user', a['user_id'])}" title="${a['username']}">${a['friendly_name']}</a>
             </div>
             % else:
             <div class="library-user-instance-box" style="background-image: url(${a['user_thumb']});"></div>

data/interfaces/default/login.html

@@ -24,7 +24,7 @@
 
     <!-- ICONS -->
     <!-- Android -->
-    <link rel="manifest" href="${http_root}images/favicon/manifest.json?v=2.0.5" crossorigin="use-credentials>
+    <link rel="manifest" href="${http_root}images/favicon/manifest.json?v=2.0.5" crossorigin="use-credentials">
     <meta name="theme-color" content="#282a2d">
     <!-- Apple -->
     <link rel="apple-touch-icon" sizes="180x180" href="${http_root}images/favicon/apple-touch-icon.png?v=2.0.5">

data/interfaces/default/mobile_device_config.html

@@ -33,7 +33,7 @@
                                 <label for="friendly_name">OneSignal Device ID</label>
                                 <div class="row">
                                     <div class="col-md-8">
-                                        <input type="text" class="form-control" id="device_id" value="${device['device_id']}" size="30" readonly>
+                                        <input type="text" class="form-control" id="onesignal_id" value="${device['onesignal_id'] or ''}" size="30" readonly>
                                     </div>
                                 </div>
                                 <p class="help-block">Your OneSignal device ID for notifications.</p>

data/interfaces/default/mobile_devices_table.html

@@ -13,7 +13,11 @@ DOCUMENTATION :: END
     % for device in sorted(devices_list, key=lambda k: k['device_name']):
     <li class="mobile-device pointer" data-id="${device['id']}" data-name="${device['device_name']}">
         <span>
+            % if device['official']:
                 <span class="toggle-left"><i class="fa fa-lg fa-fw fa-mobile"></i></span>
+            % else:
+                <span class="toggle-left officail-tooltip" data-toggle="tooltip" data-placement="top" title="Unofficial or Unknown App"><i class="fa fa-lg fa-fw fa-exclamation-triangle"></i></span>
+            % endif
             ${device['friendly_name'] or device['device_name']} &nbsp;<span class="friendly_name">(${device['id']})</span>
             <span class="toggle-right"><i class="fa fa-lg fa-fw fa-cog"></i></span>
             <span class="toggle-right friendly_name" id="device-last_seen-${device['id']}">
@@ -117,6 +121,7 @@ DOCUMENTATION :: END
     });
 
     $('#api_qr_address').change(function () {
+        this.value = $.trim(this.value);
         var url = $(this).val();
         checkQRAddress(url);
 
@@ -138,4 +143,6 @@ DOCUMENTATION :: END
         }
         verifiedDevice = true;
     })
+
+    $('.officail-tooltip').tooltip();
 </script>

data/interfaces/default/newsletter_config.html

@@ -123,7 +123,7 @@
                                         <div class="row">
                                             <div class="col-md-12">
                                                 <select class="form-control" id="${item['name']}" name="${item['name']}">
-                                                    % for key, value in sorted(item['select_options'].iteritems()):
+                                                    % for key, value in sorted(item['select_options'].items()):
                                                     % if key == item['value']:
                                                     <option value="${key}" selected>${value}</option>
                                                     % else:
@@ -144,7 +144,7 @@
                                                     <option value="select-all">Select All</option>
                                                     <option value="remove-all">Remove All</option>
                                                     % if isinstance(item['select_options'], dict):
-                                                    % for section, options in item['select_options'].iteritems():
+                                                    % for section, options in item['select_options'].items():
                                                     <optgroup label="${section}">
                                                         % for option in sorted(options, key=lambda x: x['text'].lower()):
                                                             <option value="${option['value']}">${option['text']}</option>
@@ -325,7 +325,7 @@
                                             <div class="row">
                                                 <div class="col-md-12">
                                                     <select class="form-control" id="${item['name']}" name="${item['name']}">
-                                                        % for key, value in sorted(item['select_options'].iteritems()):
+                                                        % for key, value in sorted(item['select_options'].items()):
                                                         % if key == item['value']:
                                                         <option value="${key}" selected>${value}</option>
                                                         % else:
@@ -346,7 +346,7 @@
                                                         <option value="select-all">Select All</option>
                                                         <option value="remove-all">Remove All</option>
                                                         % if isinstance(item['select_options'], dict):
-                                                        % for section, options in item['select_options'].iteritems():
+                                                        % for section, options in item['select_options'].items():
                                                         <optgroup label="${section}">
                                                             % for option in sorted(options, key=lambda x: x['text'].lower()):
                                                                 <option value="${option['value']}">${option['text']}</option>

data/interfaces/default/newsletter_preview.html

@@ -1,5 +1,5 @@
 <%
-    import urllib
+    from six.moves.urllib.parse import urlencode
 %>
 <!doctype html>
 
@@ -35,7 +35,7 @@
 <script>
 $(document).ready(function () {
     var frame = $('<iframe></iframe>', {
-        src: 'real_newsletter?${urllib.urlencode(kwargs) | n}',
+        src: 'real_newsletter?${urlencode(kwargs) | n}',
         frameborder: '0',
         style: 'display: none; height: 100vh; width: 100vw;'
     });

data/interfaces/default/newsletters_table.html

@@ -9,7 +9,7 @@ Version:            0.1
 DOCUMENTATION :: END
 </%doc>
 
-<% from plexpy.newsletter_handler import NEWSLETTER_SCHED %>
+<% from plexpy import newsletter_handler %>
 <ul class="stacked-configs list-unstyled">
     % for newsletter in sorted(newsletters_list, key=lambda k: (k['agent_label'], k['friendly_name'], k['id'])):
     <li class="newsletter-agent pointer" data-id="${newsletter['id']}">
@@ -22,8 +22,8 @@ DOCUMENTATION :: END
             % endif
             <span class="toggle-right"><i class="fa fa-lg fa-fw fa-cog"></i></span>
             <span class="toggle-right friendly_name" id="newsletter-next_run-${newsletter['id']}">
-                % if NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])):
+                % if newsletter_handler.NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])):
-                <% job = NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])) %>
+                <% job = newsletter_handler.NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])) %>
                 <script>
                     $("#newsletter-next_run-${newsletter['id']}").text(moment("${job.next_run_time}", "YYYY-MM-DD HH:mm:ssZ").fromNow())
                 </script>

data/interfaces/default/notifier_config.html

@@ -49,7 +49,16 @@
                                         <label for="${item['name']}">${item['label']}</label>
                                         <div class="row">
                                             <div class="col-md-12">
+                                                % if notifier['agent_name'] == 'scripts' and item['name'] == 'scripts_script_folder':
+                                                <div class="input-group">
                                                     <input type="${item['input_type']}" class="form-control" id="${item['name']}" name="${item['name']}" value="${item['value']}" size="30" ${'readonly' if item.get('readonly') else ''}>
+                                                    <span class="input-group-btn">
+                                                        <button class="btn btn-form" type="button" id="${item['name']}_browse" data-toggle="browse" data-filter=".folderonly" data-target="#${item['name']}">Browse</button>
+                                                    </span>
+                                                </div>
+                                                % else:
+                                                <input type="${item['input_type']}" class="form-control" id="${item['name']}" name="${item['name']}" value="${item['value']}" size="30" ${'readonly' if item.get('readonly') else ''}>
+                                                % endif
                                             </div>
                                         </div>
                                         <p class="help-block">${item['description'] | n}</p>
@@ -88,7 +97,7 @@
                                         <div class="row">
                                             <div class="col-md-12">
                                                 <select class="form-control" id="${item['name']}" name="${item['name']}">
-                                                    % for key, value in sorted(item['select_options'].iteritems()):
+                                                    % for key, value in sorted(item['select_options'].items()):
                                                     % if key == item['value']:
                                                     <option value="${key}" selected>${value}</option>
                                                     % else:
@@ -109,7 +118,7 @@
                                                     <option value="select-all">Select All</option>
                                                     <option value="remove-all">Remove All</option>
                                                     % if isinstance(item['select_options'], dict):
-                                                    % for section, options in item['select_options'].iteritems():
+                                                    % for section, options in item['select_options'].items():
                                                     <optgroup label="${section}">
                                                         % for option in sorted(options, key=lambda x: x['text'].lower()):
                                                             <option value="${option['value']}">${option['text']}</option>
@@ -211,7 +220,7 @@
                                 % for action in available_notification_actions:
                                 <li>
                                     <div class="link">
-                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>&nbsp;
+                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>
                                         ${action['label']}
                                         <span class="toggle-right"><i class="fa fa-chevron-down"></i></span>
                                     </div>
@@ -237,7 +246,7 @@
                                 % for action in available_notification_actions:
                                 <li>
                                     <div class="link">
-                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>&nbsp;
+                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>
                                         ${action['label']}
                                         <span class="toggle-right"><i class="fa fa-chevron-down"></i></span>
                                     </div>
@@ -268,7 +277,7 @@
                                 % for action in available_notification_actions:
                                 <li>
                                     <div class="link">
-                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>&nbsp;
+                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>
                                         ${action['label']}
                                         <span class="toggle-right"><i class="fa fa-chevron-down"></i></span>
                                     </div>
@@ -313,7 +322,7 @@
                                 <div class="row">
                                     <div class="col-md-12">
                                         <select class="form-control" id="test_script" name="test_script">
-                                            % for key, value in sorted(notifier['config_options'][2]['select_options'].iteritems()):
+                                            % for key, value in sorted(notifier['config_options'][2]['select_options'].items()):
                                             <option value="${key}">${value}</option>
                                             % endfor
                                         </select>
@@ -853,10 +862,8 @@
                 PNotify.prototype.options.hide = true;
                 PNotify.prototype.options.delay = $('#browser_auto_hide_delay').val() * 1000;
             }
-            var notification = new PNotify({
+            displayPNotify($('#test_subject').val(), $('#test_body').val());
-                title: $('#test_subject').val(),
+            showMsg('<i class="fa fa-check"></i> Notification sent.', false, true, 5000);
-                text: $('#test_body').val()
-            });
         }
     }
 

data/interfaces/default/recently_added.html

@@ -32,7 +32,7 @@ DOCUMENTATION :: END
 
 % if data != None:
 <%
-    from plexpy.helpers import page
+    from plexpy.helpers import cast_to_int, page
 %>
 % if data:
 <div class="dashboard-recent-media-row">
@@ -87,7 +87,7 @@ DOCUMENTATION :: END
                             <a href="${page('info', item['rating_key'])}" title="${item['title']}">${item['title']}</a>
                         </h3>
                         <h3 class="text-muted">
-                            ${item['child_count']} Seasons
+                            ${item['child_count']} Season${'s' if cast_to_int(item['child_count']) > 1 else ''}
                         </h3>
                         <h3 class="text-muted">&nbsp;</h3>
                     </div>

data/interfaces/default/scheduler_table.html

@@ -28,7 +28,7 @@ DOCUMENTATION :: END
         </tr>
     </thead>
     <tbody>
-        % for job in common.SCHEDULER_LIST:
+        % for job, job_type in common.SCHEDULER_LIST.items():
         % if job in scheduled_jobs:
         <%
             sched_job = plexpy.SCHED.get_job(job)
@@ -41,12 +41,12 @@ DOCUMENTATION :: END
             <td>${helpers.format_timedelta_Hms(sched_job.next_run_time - now)}</td>
             <td>${sched_job.next_run_time.astimezone(plexpy.SYS_TIMEZONE).strftime('%Y-%m-%d %H:%M:%S')}</td>
         </tr>
-        % elif job in ('Check for server response', 'Check for active sessions', 'Check for recently added items') and plexpy.WS_CONNECTED:
+        % elif job_type == 'websocket' and plexpy.WS_CONNECTED:
         <tr>
             % if job == 'Check for active sessions':
-            <td><a class="queue-modal-link" href="#" data-queue="active sessions">${job}</a></td>
+            <td><a class="queue-modal-link no-highlight" href="#" data-queue="active sessions">${job}</a></td>
             % elif job == 'Check for recently added items':
-            <td><a class="queue-modal-link" href="#" data-queue="recently added">${job}</a></td>
+            <td><a class="queue-modal-link no-highlight" href="#" data-queue="recently added">${job}</a></td>
             % else:
             <td>${job}</td>
             % endif

data/interfaces/default/settings.html

@@ -8,7 +8,7 @@
     from plexpy.helpers import anon_url, checked
 
     docker_setting = 'disabled' if plexpy.DOCKER else ''
-    docker_msg = '<span class="docker-setting small">(Controlled by Docker Container)</span>' if plexpy.DOCKER else ''
+    docker_msg = '<span class="setting-message small">(Controlled by Docker Container)</span>' if plexpy.DOCKER else ''
 
     available_notification_agents = sorted(notifiers.available_notification_agents(), key=lambda k: k['label'].lower())
     available_newsletter_agents = sorted(newsletters.available_newsletter_agents(), key=lambda k: k['label'].lower())
@@ -71,6 +71,13 @@
                             <h3>Version ${common.RELEASE} <small><a id="changelog-modal-link" href="#"><i class="fa fa-info-circle"></i> Changelog</a></small></h3>
                         </div>
                         % endif
+                        <div class="padded-header">
+                            <h3>Tautulli News</h3>
+                        </div>
+                        <div id="tautulli-news">
+                            <div class='text-muted'><i class="fa fa-refresh fa-spin"></i> Loading news...</div>
+                            <br>
+                        </div>
                         <div class="padded-header">
                             <h3>Tautulli Configuration</h3>
                         </div>
@@ -215,12 +222,14 @@
                             <p class="help-block">Check for Tautulli updates periodically.</p>
                         </div>
                         <div id="git_update_options">
+                            % if not plexpy.FROZEN:
                             <div class="checkbox">
                                 <label>
                                     <input type="checkbox" id="plexpy_auto_update" name="plexpy_auto_update" value="1" ${config['plexpy_auto_update']} ${docker_setting}> Update Automatically ${docker_msg | n}
                                 </label>
                                 <p class="help-block">Update Tautulli automatically if an update is available.</p>
                             </div>
+                            % endif
                             <div class="form-group advanced-setting">
                                 <label for="git_token">GitHub API Token</label>
                                 <div class="row">
@@ -448,12 +457,27 @@
                         </div>
 
                         <p class="help-block">Note: Web interface changes require a restart.</p>
-                        % if os.name == 'nt':
+                        % if common.PLATFORM in ('Windows', 'Darwin'):
+                        <%
+                            tray = {'Windows': 'System Tray', 'Darwin': 'Menu Bar'}
+                            tray_disabled = tray_disabled_msg = ''
+                            if common.PLATFORM == 'Darwin':
+                                from plexpy.macos import HAS_PYOBJC
+                                if not HAS_PYOBJC:
+                                    tray_disabled = 'disabled'
+                                    tray_disabled_msg = '<span class="setting-message small">(Missing pyobjc module)</span>'
+                        %>
                         <div class="checkbox">
                             <label>
-                                <input type="checkbox" class="http-settings" name="win_sys_tray" id="win_sys_tray" value="1" ${config['win_sys_tray']}> Enable System Tray Icon
+                                <input type="checkbox" class="http-settings" name="sys_tray_icon" id="sys_tray_icon" value="1" ${config['sys_tray_icon']} ${tray_disabled}> Enable ${tray[common.PLATFORM]} Icon ${tray_disabled_msg | n}
                             </label>
-                            <p class="help-block">Show Tautulli shortcut in the system tray.</p>
+                            <p class="help-block">Show Tautulli shortcut in the ${tray[common.PLATFORM].lower()}.</p>
+                        </div>
+                        <div class="checkbox">
+                            <label>
+                                <input type="checkbox" name="launch_startup" id="launch_startup" value="1" ${config['launch_startup']}> Launch at System Startup
+                            </label>
+                            <p class="help-block">Start Tautulli automatically after Login.</p>
                         </div>
                         % endif
                         <div class="checkbox">
@@ -544,29 +568,44 @@
                             <div class="form-group advanced-setting">
                                 <label for="https_cert">HTTPS Certificate</label>
                                 <div class="row">
-                                    <div class="col-md-6">
+                                    <div class="col-md-7">
+                                        <div class="input-group">
                                             <input type="text" class="form-control http-settings" id="https_cert" name="https_cert" value="${config['https_cert']}">
+                                            <span class="input-group-btn">
+                                                <button class="btn btn-form" type="button" id="https_cert_browse" data-toggle="browse" data-filter=".pem" data-target="#https_cert">Browse</button>
+                                            </span>
                                         </div>
                                     </div>
-                                <p class="help-block">The location of the SSL certificate.</p>
+                                </div>
+                                <p class="help-block">The location of the SSL certificate in PEM format.</p>
                             </div>
                             <div class="form-group advanced-setting">
                                 <label for="https_cert_chain">HTTPS Certificate Chain</label>
                                 <div class="row">
-                                    <div class="col-md-6">
+                                    <div class="col-md-7">
+                                        <div class="input-group">
                                             <input type="text" class="form-control http-settings" id="https_cert_chain" name="https_cert_chain" value="${config['https_cert_chain']}">
+                                            <span class="input-group-btn">
+                                                <button class="btn btn-form" type="button" id="https_cert_chain_browse" data-toggle="browse" data-filter=".pem" data-target="#https_cert_chain">Browse</button>
+                                            </span>
                                         </div>
                                     </div>
-                                <p class="help-block">The location of the SSL certificate chain.</p>
+                                </div>
+                                <p class="help-block">The location of the SSL certificate chain in PEM format.</p>
                             </div>
                             <div class="form-group advanced-setting">
                                 <label for="https_key">HTTPS Key</label>
                                 <div class="row">
-                                    <div class="col-md-6">
+                                    <div class="col-md-7">
+                                        <div class="input-group">
                                             <input type="text" class="form-control http-settings" id="https_key" name="https_key" value="${config['https_key']}">
+                                            <span class="input-group-btn">
+                                                <button class="btn btn-form" type="button" id="https_key_browse" data-toggle="browse" data-filter=".pem" data-target="#https_key">Browse</button>
+                                            </span>
                                         </div>
                                     </div>
-                                <p class="help-block">The location of the SSL key.</p>
+                                </div>
+                                <p class="help-block">The location of the SSL key in PEM format.</p>
                             </div>
                         </div>
 
@@ -751,7 +790,6 @@
                             <label>
                                 <input type="checkbox" class="pms-settings" id="pms_url_manual" name="pms_url_manual" value="1" ${config['pms_url_manual']}> Manual Connection
                             </label>
-                            <span id="cloudManualConnection" style="display: none; color: #eb8600; padding-left: 10px;"> Not available for Plex Cloud servers.</span>
                             <p class="help-block">Use the user defined connection details. Do not retrieve the server connection URL automatically.</p>
                         </div>
                         <div class="form-group advanced-setting">
@@ -777,10 +815,15 @@
                         <input type="checkbox" name="server_changed" id="server_changed" value="1" style="display: none;">
 
                         <div class="form-group advanced-setting">
-                            <label for="pms_logs_folder">Logs Folder</label>
+                            <label for="pms_logs_folder">Plex Logs Folder</label>
                             <div class="row">
-                                <div class="col-md-6">
+                                <div class="col-md-7">
+                                    <div class="input-group">
                                         <input type="text" class="form-control" id="pms_logs_folder" name="pms_logs_folder" value="${config['pms_logs_folder']}" size="30" data-parsley-trigger="change" data-parsley-pattern="^[^\~\%]" data-parsley-errors-container="#pms_logs_folder_error"  data-parsley-error-message="Shortcuts are not recognized.">
+                                        <span class="input-group-btn">
+                                            <button class="btn btn-form" type="button" id="pms_logs_folder_browse" data-toggle="browse" data-filter=".folderonly" data-target="#pms_logs_folder">Browse</button>
+                                        </span>
+                                    </div>
                                 </div>
                                 <div id="pms_logs_folder_error" class="alert alert-danger settings-alert" role="alert"></div>
                             </div>
@@ -808,7 +851,6 @@
                             <label>
                                 <input type="checkbox" id="monitor_pms_updates" name="monitor_pms_updates" value="1" ${config['monitor_pms_updates']}> Monitor Plex Updates
                             </label>
-                            <span id="cloudMonitorUpdates" style="display: none; color: #eb8600; padding-left: 10px;"> Not available for Plex Cloud servers.</span>
                             <p class="help-block">Enable to have Tautulli check if updates are available for the Plex Media Server.</p>
                         </div>
                         <div id="pms_update_options">
@@ -842,36 +884,6 @@
                                 </p>
                             </div>
                         </div>
-                        <div class="checkbox">
-                            <label>
-                                <input type="checkbox" id="monitor_remote_access" name="monitor_remote_access" value="1" ${config['monitor_remote_access']}> Monitor Plex Remote Access
-                            </label>
-                            <span id="cloudMonitorRemoteAccess" style="display: none; color: #eb8600; padding-left: 10px;"> Not available for Plex Cloud servers.</span>
-                            <span id="remoteAccessCheck" class="settings-warning"></span>
-                            <p class="help-block">Enable to have Tautulli check if remote access to the Plex Media Server goes down.</p>
-                        </div>
-                        <div id="monitor_remote_access_options">
-                            <div class="form-group advanced-setting">
-                                <label for="remote_access_ping_interval">Remote Access Ping Interval</label>
-                                <div class="row">
-                                    <div class="col-md-2">
-                                        <input type="text" class="form-control" data-parsley-type="integer" id="remote_access_ping_interval" name="remote_access_ping_interval" value="${config['remote_access_ping_interval']}" size="5" data-parsley-min="60" data-parsley-trigger="change" data-parsley-errors-container="#remote_access_ping_interval_error" required>
-                                    </div>
-                                    <div id="remote_access_ping_interval_error" class="alert alert-danger settings-alert" role="alert"></div>
-                                </div>
-                                <p class="help-block">The interval (in seconds) Tautulli will ping the Plex Media Server for the remote access status. Minimum 60.</p>
-                            </div>
-                            <div class="form-group advanced-setting">
-                                <label for="remote_access_ping_threshold">Remote Access Ping Threshold</label>
-                                <div class="row">
-                                    <div class="col-md-2">
-                                        <input type="text" class="form-control" data-parsley-type="integer" id="remote_access_ping_threshold" name="remote_access_ping_threshold" value="${config['remote_access_ping_threshold']}" size="5" data-parsley-min="1" data-parsley-trigger="change" data-parsley-errors-container="#remote_access_ping_threshold_error" required>
-                                    </div>
-                                    <div id="remote_access_ping_threshold_error" class="alert alert-danger settings-alert" role="alert"></div>
-                                </div>
-                                <p class="help-block">The number of consecutive remote access status failures to consider remote access as down. Minimum 1.</p>
-                            </div>
-                        </div>
 
                         <div class="form-group advanced-setting">
                             <label for="refresh_users_interval">Users List Refresh Interval</label>
@@ -1025,14 +1037,14 @@
                             </p>
                         </div>
                         <div class="form-group">
-                            <label for="notify_recently_added_delay">Notification Delay</label>
+                            <label for="notify_recently_added_delay">Recently Added Notification Delay</label>
                             <div class="row">
                                 <div class="col-md-2">
                                     <input type="text" class="form-control" data-parsley-type="integer" id="notify_recently_added_delay" name="notify_recently_added_delay" value="${config['notify_recently_added_delay']}" size="5" data-parsley-min="60" data-parsley-trigger="change" data-parsley-errors-container="#notify_recently_added_delay_error" required>
                                 </div>
                                 <div id="notify_recently_added_delay_error" class="alert alert-danger settings-alert" role="alert"></div>
                             </div>
-                            <p class="help-block">Set the delay (in seconds) to wait for consecutive recently added items to group together and to allow metadata to be processed before sending the notification. Minimum 60 seconds.</p>
+                            <p class="help-block">Set the delay (in seconds) to wait for consecutive recently added items to group together and to allow metadata to be processed before sending the recently added notification. Minimum 60 seconds, default 300.</p>
                         </div>
                         <div class="form-group advanced-setting">
                             <label>Flush Recently Added</label>
@@ -1058,6 +1070,21 @@
                             </p>
                         </div>-->
 
+                        <div class="padded-header">
+                            <h3>Remote Access Notifications</h3>
+                        </div>
+
+                        <div class="form-group">
+                            <label for="notify_remote_access_threshold">Remote Access Down Threshold</label>
+                            <div class="row">
+                                <div class="col-md-2">
+                                    <input type="text" class="form-control" data-parsley-type="integer" id="notify_remote_access_threshold" name="notify_remote_access_threshold" value="${config['notify_remote_access_threshold']}" size="5" data-parsley-min="60" data-parsley-trigger="change" data-parsley-errors-container="#notify_remote_access_threshold_error" required>
+                                </div>
+                                <div id="notify_remote_access_threshold_error" class="alert alert-danger settings-alert" role="alert"></div>
+                            </div>
+                            <p class="help-block">The duration (in seconds) for Plex remote access to be down before sending a notification. Minimum 60, default 60.</p>
+                        </div>
+
                         <div class="padded-header">
                             <h3>Newsletters</h3>
                         </div>
@@ -1110,10 +1137,15 @@
                             </p>
                         </div>
                         <div class="form-group advanced-setting">
-                            <label for="newsletter_dir">Custom Newsletter Templates Folder</label>
+                            <label for="newsletter_custom_dir">Custom Newsletter Templates Folder</label>
                             <div class="row">
-                                <div class="col-md-6">
+                                <div class="col-md-7">
+                                    <div class="input-group">
                                         <input type="text" class="form-control" id="newsletter_custom_dir" name="newsletter_custom_dir" value="${config['newsletter_custom_dir']}">
+                                        <span class="input-group-btn">
+                                            <button class="btn btn-form" type="button" id="newsletter_custom_dir_browse" data-toggle="browse" data-filter=".folderonly" data-target="#newsletter_custom_dir">Browse</button>
+                                        </span>
+                                    </div>
                                 </div>
                             </div>
                             <p class="help-block">Optional: Enter the full path to your custom newsletter templates folder. Leave blank for default.</p>
@@ -1121,8 +1153,13 @@
                         <div class="form-group advanced-setting">
                             <label for="newsletter_dir">Newsletter Output Directory</label> ${docker_msg | n}
                             <div class="row">
-                                <div class="col-md-6">
+                                <div class="col-md-7">
+                                    <div class="input-group">
                                         <input type="text" class="form-control" id="newsletter_dir" name="newsletter_dir" value="${config['newsletter_dir']}" ${docker_setting}>
+                                        <span class="input-group-btn">
+                                            <button class="btn btn-form" type="button" id="newsletter_dir_browse" data-toggle="browse" data-filter=".folderonly" data-target="#newsletter_dir" ${docker_setting}>Browse</button>
+                                        </span>
+                                    </div>
                                 </div>
                             </div>
                             <p class="help-block">Enter the full path to where newsletter files will be saved.</p>
@@ -1310,14 +1347,33 @@
                     <div role="tabpanel" class="tab-pane" id="tabs-import_backups">
 
                         <div class="padded-header">
-                            <h3>Database Import</h3>
+                            <h3>Import</h3>
                         </div>
 
-                        <p class="help-block">Click a button below to import an existing database from another app.</p>
+                        <div class="form-group">
+                            <label for="database_import">Database Import</label>
+                            <p class="help-block">Click a button below to import an existing database from the selected app.</p>
+                            <div class="row">
+                                <div class="col-md-9">
                                     <div class="btn-group">
+                                        <button class="btn btn-form toggle-app-import-modal" type="button" data-target="#app-import-modal" data-toggle="modal" data-app="tautulli">Tautulli</button>
                                         <button class="btn btn-form toggle-app-import-modal" type="button" data-target="#app-import-modal" data-toggle="modal" data-app="plexwatch">PlexWatch</button>
                                         <button class="btn btn-form toggle-app-import-modal" type="button" data-target="#app-import-modal" data-toggle="modal" data-app="plexivity">Plexivity</button>
                                     </div>
+                                </div>
+                            </div>
+                        </div>
+                        <div class="form-group">
+                            <label for="config_import">Configuration Import</label>
+                            <p class="help-block">Click the button below to import a previous Tautulli configuration.</p>
+                            <div class="row">
+                                <div class="col-md-9">
+                                    <div class="btn-group">
+                                        <button class="btn btn-form toggle-config-import-modal" type="button" data-target="#config-import-modal" data-toggle="modal">Tautulli</button>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
 
                         <div class="padded-header">
                             <h3>Backup</h3>
@@ -1354,8 +1410,13 @@
                         <div class="form-group">
                             <label for="log_dir">Log Directory</label> ${docker_msg | n}
                             <div class="row">
-                                <div class="col-md-6">
+                                <div class="col-md-7">
+                                    <div class="input-group">
                                         <input type="text" class="form-control directory-settings" id="log_dir" name="log_dir" value="${config['log_dir']}" ${docker_setting}>
+                                        <span class="input-group-btn">
+                                            <button class="btn btn-form" type="button" id="log_dir_browse" data-toggle="browse" data-filter=".folderonly" data-target="#log_dir" ${docker_setting}>Browse</button>
+                                        </span>
+                                    </div>
                                     <div class="btn-group">
                                         <button class="btn btn-form" type="button" id="clear_logs">Clear Logs</button>
                                     </div>
@@ -1365,8 +1426,13 @@
                         <div class="form-group">
                             <label for="backup_dir">Backup Directory</label> ${docker_msg | n}
                             <div class="row">
-                                <div class="col-md-6">
+                                <div class="col-md-7">
+                                    <div class="input-group">
                                         <input type="text" class="form-control directory-settings" id="backup_dir" name="backup_dir" value="${config['backup_dir']}" ${docker_setting}>
+                                        <span class="input-group-btn">
+                                            <button class="btn btn-form" type="button" id="backup_dir_browse" data-toggle="browse" data-filter=".folderonly" data-target="#backup_dir" ${docker_setting}>Browse</button>
+                                        </span>
+                                    </div>
                                     <div class="btn-group">
                                         <button class="btn btn-form" type="button" id="backup_config">Backup Config</button>
                                         <button class="btn btn-form" type="button" id="backup_database">Backup Database</button>
@@ -1377,8 +1443,13 @@
                         <div class="form-group">
                             <label for="cache_dir">Cache Directory</label> ${docker_msg | n}
                             <div class="row">
-                                <div class="col-md-6">
+                                <div class="col-md-7">
+                                    <div class="input-group">
                                         <input type="text" class="form-control directory-settings" id="cache_dir" name="cache_dir" value="${config['cache_dir']}" ${docker_setting}>
+                                        <span class="input-group-btn">
+                                            <button class="btn btn-form" type="button" id="cache_dir_browse" data-toggle="browse" data-filter=".folderonly" data-target="#cache_dir" ${docker_setting}>Browse</button>
+                                        </span>
+                                    </div>
                                     <div class="btn-group">
                                         <button class="btn btn-form" type="button" id="clear_cache">Clear All Cache</button>
                                         <button class="btn btn-form" type="button" id="clear_image_cache">Clear Image Cache</button>
@@ -1410,6 +1481,7 @@
                             <label>Registered Devices</label>
                             <p class="help-block">Register a new device using a QR code, or configure an existing device by clicking the settings icon on the right.</p>
                             <p id="app_api_msg" style="color: #eb8600;">Warning: The API must be enabled under <a data-tab-destination="web_interface" data-target="api_enabled">Web Interface</a> to use the app.</p>
+                            <br />
                             <div class="row">
                                 <div id="plexpy-mobile-devices-table" class="col-md-12">
                                     <div class='text-muted'><i class="fa fa-refresh fa-spin"></i> Loading registered devices...</div>
@@ -1506,6 +1578,7 @@
     </div>
 </div>
 <div id="app-import-modal" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="app-import-modal"></div>
+<div id="config-import-modal" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="config-import-modal"></div>
 <div id="add-notifier-modal" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="add-notifier-modal">
     <div class="modal-dialog" role="document">
         <div class="modal-content">
@@ -1845,7 +1918,10 @@ Rating: {rating}/10   --> Rating: /10
                 <label>Instructions</label>
                 <p class="help-block">
                     Scan the QR code below with the Tautulli Android app to automatically register it with the server (make sure the Tautulli Address below is correct)
-                    or manually enter the connection info and device token into the app settings.
+                    or manually enter the connection info and device token into the app settings. This window will automatically close once device registration is successful.
+                </p>
+                <p class="help-block">
+                    Note: OneSignal.com must not be blocked (e.g. in Pi-hole) for device registration.
                 </p>
                 <label>QR Code</label>
                 <pre id="api_qr_code" style="text-align: center"></pre>
@@ -1857,7 +1933,7 @@ Rating: {rating}/10   --> Rating: /10
                 </p>
                 <p class="help-block" id="api_qr_private" style="display: none;">
                     Note: This is a private IP address. Tautulli will not be reachable outside of your home network.
-                    Access Tautulli via an externally address or manually enter the address above to generate the QR code for remote access.
+                    Access Tautulli via an external address or manually enter the address above to generate the QR code for remote access.
                 </p>
                 <p class="help-block" id="api_qr_https" style="display: none;">
                     Note: This URL is not secure. Requests between the app and the server will not be encrypted.
@@ -1873,6 +1949,38 @@ Rating: {rating}/10   --> Rating: /10
     </div>
 </div>
 <div id="mobile-device-config-modal" class="modal fade wide" tabindex="-1" role="dialog" aria-labelledby="mobile-device-config-modal"></div>
+<div id="browse-path-modal" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="browse-path-modal">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-hidden="true"><i class="fa fa-remove"></i></button>
+                <h4 class="modal-title">File Browser</h4>
+            </div>
+            <div class="modal-body">
+                <div class="row">
+                    <div class="col-md-12">
+                        <div class="form-group">
+                            <label for="browse-path">Select a <span id="browse-path-type"></span> Below</label>
+                            <div class="row">
+                                <div class="col-md-12">
+                                    <input type="text" class="form-control" id="browse-path" name="browse-path" value="" size="30" disabled>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                    <div class="col-md-12" style="height: 400px; overflow: auto;">
+                        <ul id="browse-path-list" class="stacked-configs list-unstyled">
+                        </ul>
+                    </div>
+                </div>
+            </div>
+            <div class="modal-footer">
+                <span id="browse-path-status-message" style="padding-right: 25px;"></span>
+                <input type="button" id="select-browse-file" class="btn btn-bright" value="Select">
+            </div>
+        </div>
+    </div>
+</div>
 </%def>
 
 <%def name="javascriptIncludes()">
@@ -1979,6 +2087,54 @@ Rating: {rating}/10   --> Rating: /10
         });
     }
 
+    $("#browse-path-modal").on('hidden.bs.modal', function() {
+        $("#select-browse-file").unbind('click');
+    });
+
+    function openBrowsePath(key, path, filter_ext, file_description, select_target) {
+        $("#browse-path-type").text(file_description);
+        $("#browse-path-modal").modal('show');
+
+        $("#select-browse-file").click(function () {
+            $("#browse-path-modal").modal('hide');
+            $(select_target).val($("#browse-path").val()).change();
+        });
+
+        browsePath(key, path, filter_ext);
+    }
+
+    function browsePath(key, path, filter_ext) {
+        $("#browse-path-status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>');
+        getBrowsePath(key, path, filter_ext).then(function (data) {
+            if (data.result === 'error') {
+                $("#browse-path-status-message").html("<i class='fa fa-exclamation-triangle'></i> " + data.message);
+            } else {
+                $("#browse-path-status-message").html("");
+
+                $('#browse-path').val(data.path);
+                var browse_list = $('#browse-path-list');
+                browse_list.parent().animate({ scrollTop: 0 }, 0);
+                browse_list.empty();
+
+                $.each(data.data, function(i, item) {
+                    var browse_item = $('<li/>')
+                        .html("<span><i class='fa fa-fw fa-" + item.icon + "'></i>&nbsp; " + item.title + "</span>")
+                        .addClass(item.type + ' pointer')
+                        .data('key', item.key)
+                        .data('path', item.path)
+                        .appendTo(browse_list)
+                });
+
+                $('#browse-path-list li').click(function (){
+                    $('#browse-path').val($(this).data('path'));
+                    if ($(this).hasClass('folder')) {
+                        browsePath($(this).data('key'), null, filter_ext)
+                    }
+                });
+            }
+        });
+    }
+
 $(document).ready(function() {
 
     // Javascript to enable link to tab
@@ -2065,7 +2221,6 @@ $(document).ready(function() {
     initConfigCheckbox('#https_create_cert');
     initConfigCheckbox('#check_github');
     initConfigCheckbox('#monitor_pms_updates');
-    initConfigCheckbox('#monitor_remote_access');
     initConfigCheckbox('#newsletter_self_hosted');
 
     $('#menu_link_shutdown').click(function() {
@@ -2311,7 +2466,6 @@ $(document).ready(function() {
             $('#pms_is_cloud').val(is_cloud !== 'undefined' && is_cloud === true ? 1 : 0);
             $('#pms_url_manual').prop('checked', false);
             $('#pms_url').val('Please verify your server above to retrieve the URL');
-            PMSCloudCheck();
         },
         onDropdownOpen: function() {
             this.clear();
@@ -2342,38 +2496,6 @@ $(document).ready(function() {
     }
     getServerOptions();
 
-    function PMSCloudCheck() {
-        if ($('#pms_is_cloud').val() === "1") {
-            $('#pms_port').val(443).prop('readonly', true);
-            $('#pms_is_remote_checkbox').prop('checked', true).prop('disabled', true);
-            $('#pms_is_remote').val(1);
-            $('#pms_ssl_checkbox').prop('checked', true).prop('disabled', true);
-            $('#pms_ssl').val(1);
-            $('#pms_url_manual').prop('checked', false).prop('disabled', true);
-            $('#monitor_pms_updates').prop('checked', false).prop('disabled', true);
-            $('#pms_update_options').hide();
-            $('#monitor_remote_access').prop('checked', false).prop('disabled', true);
-            $('#cloudManualConnection').show();
-            $('#cloudMonitorUpdates').show();
-            $('#cloudMonitorRemoteAccess').show();
-            $('#remoteAccessCheck').hide();
-        } else {
-            $('#pms_port').prop('readonly', false);
-            $('#pms_is_remote_checkbox').prop('disabled', false);
-            $('#pms_is_remote').val($('#pms_is_remote_checkbox').is(':checked') ? 1 : 0);
-            $('#pms_ssl_checkbox').prop('disabled', false);
-            $('#pms_ssl').val($('#pms_ssl_checkbox').is(':checked') ? 1 : 0);
-            $('#pms_url_manual').prop('disabled', false);
-            $('#monitor_pms_updates').prop('disabled', false);
-            $('#monitor_remote_access').prop('disabled', false);
-            $('#cloudManualConnection').hide();
-            $('#cloudMonitorUpdates').hide();
-            $('#cloudMonitorRemoteAccess').hide();
-            remoteAccessEnabledCheck()
-        }
-    }
-    PMSCloudCheck();
-
     function verifyServer(_callback) {
         var pms_ip = $("#pms_ip").val();
         var pms_port = $("#pms_port").val();
@@ -2461,9 +2583,7 @@ $(document).ready(function() {
         $("#token_verify").html('<i class="fa fa-refresh fa-spin"></i>').fadeIn('fast');
     }
     function OAuthSuccessCallback(authToken) {
-        var x_plex_headers = getPlexHeaders();
         $("#pms_token").val(authToken);
-        $("#pms_uuid").val(x_plex_headers['X-Plex-Client-Identifier']);
         $("#token_verify").html('<i class="fa fa-check"></i>').fadeIn('fast');
         getServerOptions(authToken);
     }
@@ -2488,25 +2608,22 @@ $(document).ready(function() {
         });
     });
 
+    // Load config import modal
+    $(".toggle-config-import-modal").click(function() {
+        $.ajax({
+            url: 'import_config_tool',
+            cache: false,
+            async: true,
+            complete: function(xhr, status) {
+                $("#config-import-modal").html(xhr.responseText);
+            }
+        });
+    });
+
     pms_version = false;
     pms_logs_debug = false;
     pms_logs = false;
 
-    function remoteAccessEnabledCheck() {
-        $.ajax({
-            url: 'get_server_pref',
-            data: { pref: 'PublishServerOnPlexOnlineKey' },
-            async: true,
-            success: function(data) {
-                if (data === 'false' || data === '0') {
-                    $("#remoteAccessCheck").html("Remote access must be enabled on your Plex Server. <a target='_blank' href='${anon_url('https://support.plex.tv/hc/en-us/articles/200484543-Enabling-Remote-Access-for-a-Server')}'>Click here</a> for help.");
-                    $("#monitor_remote_access").attr("checked", false).attr("disabled", true);
-                }
-            }
-        });
-    }
-    remoteAccessEnabledCheck();
-
     // Sortable home_sections
     function set_home_sections() {
         var home_sections = [];
@@ -2917,6 +3034,56 @@ $(document).ready(function() {
     $('#resources-xml').on('tripleclick', function () {
         openPlexXML('/api/resources', true, {includeHttps: 1});
     });
+
+    var tautulli_news = $('#tautulli-news')
+    $.ajax({
+        url: 'https://tautulli.com/news/tautulli-news.json',
+        type: 'GET',
+        dataType: 'json',
+        cache: false,
+        async: true,
+        success: function (data) {
+            if (data) {
+                var now = moment().endOf('day');
+                var news = $('<ul/>').addClass('accordion list-unstyled')
+                $.each(data, function (index, news_item) {
+                    var date = moment(news_item.date, "YYYY-MM-DD");
+                    if (index >= 5) { return false; }
+                    var header = $('<div/>').addClass('link').html(
+                        '<span class="toggle-left"><i class="fa fa-newspaper fa-fw"></i></span>' +
+                        '<span class="news-title">' + news_item.title + '</span>' +
+                        '<span class="toggle-right"><i class="fa fa-chevron-down fa-fw"></i></span>' +
+                        '<span class="news-date toggle-right">' + date.format($('#date_format').val()) + '</span>');
+                    var subtitle = $('<span/>').addClass('news-subtitle').html(news_item.subtitle);
+                    var body = $('<span/>').addClass('news-body').html(news_item.body);
+                    var content = $('<div/>').addClass('submenu');
+                    if (news_item.subtitle) { content.append(subtitle); }
+                    content.append(body);
+                    var li = $('<li/>').append(header).append(content)
+                    if (index === 0 && Math.abs(now.diff(date, 'days')) < 7) {
+                        li.addClass('open');
+                        content.css('display', 'block');
+                    }
+                    news.append(li)
+                });
+                tautulli_news.html(news);
+                var accordion_news = new Accordion(news, false);
+            } else {
+                tautulli_news.html('<p class="help-block"><i class="fa fa-check"></i>&nbsp; No news available.</p>')
+            }
+        },
+        error: function () {
+            tautulli_news.html('<p class="help-block"><i class="fa fa-exclamation-triangle"></i>&nbsp; Failed to retrieve news.</p>')
+        }
+    });
+
+    $("body").on('click', '[data-toggle=browse]', function () {
+        var filter = $(this).data('filter');
+        var target = $(this).data('target');
+        var path = $(target).val();
+        var description = $(this).data('description') || $("label[for='" + target.replace('#', '') + "']").text();
+        openBrowsePath(null, path, filter, description, target);
+    });
 });
 </script>
 </%def>

data/interfaces/default/shutdown.html

@@ -22,10 +22,10 @@
             <div class="modal-body" id="modal-text">
                 <div align="center">
                     % if message == "Shutting Down":
-                    <h3><i class="fa fa-refresh fa-spin"></i> Tautulli is ${message}.</h3>
+                    <h3><i class="fa fa-refresh fa-spin"></i>&nbsp; Tautulli is ${message.lower()}</h3>
                     <br />
                     % else:
-                    <h3><i class="fa fa-refresh fa-spin"></i> Tautulli is ${message}.</h3>
+                    <h3><i class="fa fa-refresh fa-spin"></i>&nbsp; Tautulli is ${message.lower()}</h3>
                     <br />
                     <h4>Restart in <span class="countdown"></span></h4>
                     % endif

data/interfaces/default/update_metadata.html

@@ -171,6 +171,7 @@ DOCUMENTATION :: END
                 </p>
                 <p> with </p>
                 <p><span id="new_title"></span></p>
+                <p>from the <strong><span id="new_library"></span></strong> library?</p>
                 % if query['media_type'] != 'movie':
                 <p>All items for <strong>${query['grandparent_title']}</strong> will also be updated.</p>
                 % endif
@@ -211,10 +212,12 @@ DOCUMENTATION :: END
 
     $(document).on('click', '#search-results-list a', function (e) {
         e.preventDefault();
-        var new_rating_key = $(this).attr('id');
+        var new_rating_key = $(this).data('rating_key');
+        var new_library_section = $(this).data('library_name');
         var new_href = $(this).attr('href');
 
         $('#new_title').html($(this).find('.item-children-instance-text-wrapper').html());
+        $('#new_library').text(new_library_section);
 
         $('#confirm-modal-update').modal();
         $('#confirm-modal-update').one('click', '#confirm-update', function () {

data/interfaces/default/user.html

@@ -284,7 +284,8 @@ DOCUMENTATION :: END
                                         <table class="display user_ip_table" id="user_ip_table-UID-${data['user_id']}" width="100%">
                                             <thead>
                                                 <tr>
-                                                    <th align="left" id="last_seen">Last Seen</th>
+                                                    <th align="left" id="last_seen">Last Streamed</th>
+                                                    <th align="left" id="first_seen">First Streamed</th>
                                                     <th align="left" id="ip_address">IP Address</th>
                                                     <th align="left" id="platform">Last Platform</th>
                                                     <th align="left" id="player">Last Player</th>

data/interfaces/default/user_player_stats.html

@@ -27,7 +27,7 @@ DOCUMENTATION :: END
             <div id="user-player-image-${a['result_id']}">
                 <div class="user-player-instance-box svg-icon platform-${a['platform_name']}"></div>
             </div>
-            <div class="user-player-instance-name">
+            <div class="user-player-instance-name" title="${a['player_name']}">
                 ${a['player_name']}
             </div>
             <div class="user-player-instance-playcount">

data/interfaces/default/users.html

@@ -34,7 +34,7 @@
                 <th align="left" id="edit_row">Edit</th>
                 <th align="right" id="avatar"></th>
                 <th align="left" id="friendly_name">User</th>
-                <th align="left" id="last_seen">Last Seen</th>
+                <th align="left" id="last_seen">Last Streamed</th>
                 <th align="left" id="last_known_ip">Last Known IP</th>
                 <th align="left" id="last_platform">Last Platform</th>
                 <th align="left" id="last_player">Last Player</th>

data/interfaces/default/welcome.html

@@ -203,7 +203,7 @@
                     <h3>Database Import</h3>
                     <div class="wizard-input-section">
                         <p class="help-block">
-                            If you have an existing PlexWatch/Plexivity database, you can import the data into Tautulli.
+                            If you have an existing Tautulli, PlexWatch, or Plexivity database, you can import the data into Tautulli.
                         </p>
                         <p class="help-block">
                             To import a database, navigate to the <strong>Settings</strong> page
@@ -216,7 +216,8 @@
                         <input type="checkbox" name="first_run" id="first_run" value="1" checked>
                         <input type="checkbox" name="group_history_tables" id="group_history_tables" value="1" checked>
                         <input type="checkbox" name="history_table_activity" id="history_table_activity" value="1" checked>
-                        <input type="checkbox" name="win_sys_tray" id="win_sys_tray" value="1" checked>
+                        <input type="checkbox" name="sys_tray_icon" id="sys_tray_icon" value="1" checked>
+                        <input type="checkbox" name="launch_startup" id="launch_startup" value="1" checked>
                         <input type="checkbox" name="launch_browser" id="launch_browser" value="1" checked>
                         <input type="checkbox" name="api_enabled" id="api_enabled" value="1" checked>
                         <input type="checkbox" name="refresh_users_on_startup" id="refresh_users_on_startup" value="1" checked>
@@ -494,7 +495,7 @@ $(document).ready(function() {
             var pms_ssl = $("#pms_ssl").val();
             var pms_is_remote = $("#pms_is_remote").val();
             if ((pms_ip !== '') || (pms_port !== '')) {
-                $("#pms-verify-status").html('<i class="fa fa-refresh fa-spin"></i>&nbsp; Validating server...');
+                $("#pms-verify-status").html('<i class="fa fa-refresh fa-spin"></i>&nbsp; Verifying server...');
                 $('#pms-verify-status').fadeIn('fast');
                 $.ajax({
                     url: 'get_server_id',
@@ -509,7 +510,7 @@ $(document).ready(function() {
                     async: true,
                     timeout: 5000,
                     error: function (jqXHR, textStatus, errorThrown) {
-                        $("#pms-verify-status").html('<i class="fa fa-exclamation-circle"></i>&nbsp; This is not a Plex Server!');
+                        $("#pms-verify-status").html('<i class="fa fa-exclamation-circle"></i>&nbsp; Error verifying server: ' + textStatus);
                         $('#pms-verify-status').fadeIn('fast');
                     },
                     success: function(xhr, status) {

init-scripts/init-alt.freebsd

@@ -1,88 +0,0 @@
-#!/bin/sh
-#
-# PROVIDE: tautulli
-# REQUIRE: tautulli
-# KEYWORD: shutdown
-#
-# Add the following lines to /etc/rc.conf.local or /etc/rc.conf
-# to enable this service:
-#
-# tautulli_enable (bool):     Set to NO by default.
-#                       Set it to YES to enable it.
-# tautulli_user:  The user account Tautulli daemon runs as what
-#                       you want it to be. It uses 'tautulli' user by
-#                       default. Do not sets it as empty or it will run
-#                       as root.
-# tautulli_dir:       Directory where Tautulli lives.
-#                       Default: /usr/local/share/Tautulli
-# tautulli_chdir:  Change to this directory before running Tautulli.
-#     Default is same as tautulli_dir.
-# tautulli_pid:  The name of the pidfile to create.
-#     Default is tautulli.pid in tautulli_dir.
-
-. /etc/rc.subr
-
-name="tautulli"
-rcvar=${name}_enable
-
-load_rc_config ${name}
-
-: ${tautulli_enable:="NO"}
-: ${tautulli_user:="tautulli"}
-: ${tautulli_dir:="/usr/local/share/Tautulli"}
-: ${tautulli_chdir:="${tautulli_dir}"}
-: ${tautulli_pid:="${tautulli_dir}/tautulli.pid"}
-: ${tautulli_conf:="${tautulli_dir}/config.ini"}
-
-WGET="/usr/local/bin/wget" # You need wget for this script to safely shutdown Tautulli.
-if [ -e "${tautulli_conf}" ]; then
-        HOST=`grep -A64 "\[General\]" "${tautulli_conf}"|egrep "^http_host"|perl -wple 's/^http_host = (.*)$/$1/'`
-        PORT=`grep -A64 "\[General\]" "${tautulli_conf}"|egrep "^http_port"|perl -wple 's/^http_port = (.*)$/$1/'`
-fi
-
-status_cmd="${name}_status"
-stop_cmd="${name}_stop"
-
-command="${tautulli_dir}/Tautulli.py"
-command_args="--daemon --quiet --nolaunch --port ${PORT} --pidfile ${tautulli_pid} --config ${tautulli_conf}"
-
-# Check for wget and refuse to start without it.
-if [ ! -x "${WGET}" ]; then
-  warn "Tautulli not started: You need wget to safely shut down Tautulli."
-  exit 1
-fi
-
-# Ensure user is root when running this script.
-if [ `id -u` != "0" ]; then
-  echo "Oops, you should be root before running this!"
-  exit 1
-fi
-
-verify_tautulli_pid() {
-    # Make sure the pid corresponds to the Tautulli process.
-    pid=`cat ${tautulli_pid} 2>/dev/null`
-    ps -p ${pid} | grep -q "python ${tautulli_dir}/Tautulli.py"
-    return $?
-}
-
-# Try to stop Tautulli cleanly by calling shutdown over http.
-tautulli_stop() {
-    if [ ! -e "${tautulli_conf}" ]; then
-        echo "Tautulli' settings file does not exist. Try starting Tautulli, as this should create the file."
-        exit 1
-    fi
-    echo "Stopping $name"
-    verify_tautulli_pid
-    ${WGET} -O - -q --user=${SBUSR} --password=${SBPWD} "http://${HOST}:${PORT}/shutdown/" >/dev/null
-    
-    if [ -n "${pid}" ]; then
-      wait_for_pids ${pid}
-      echo "Stopped $name"
-    fi
-}
-
-tautulli_status() {
-    verify_tautulli_pid && echo "$name is running as ${pid}" || echo "$name is not running"
-}
-
-run_rc_command "$1"

init-scripts/init.fedora.centos.service

@@ -1,76 +0,0 @@
-#!/bin/sh
-#
-### BEGIN INIT INFO
-# Provides:          Tautulli
-# Required-Start:    $all
-# Required-Stop:     $all
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: starts Tautulli
-# Description:       starts Tautulli
-### END INIT INFO
-
-# Source function library.
-. /etc/init.d/functions
-
-## Variables
-prog=tautulli
-lockfile=/var/lock/subsys/$prog
-homedir=/opt/Tautulli
-datadir=/opt/Tautulli
-configfile=/opt/Tautulli/config.ini
-pidfile=/var/run/tautulli.pid
-nice=
-# The following line must point to your Python 2.7 install
-python27=/usr/src/Python-2.7.11/python
-##
-
-options=" --daemon --config $configfile --pidfile $pidfile --datadir $datadir --nolaunch --quiet"
-
-start() {
-        # Start daemon.
-        echo -n $"Starting $prog: "
-        daemon --pidfile=$pidfile $nice $python27 $homedir/Tautulli.py $options
-        RETVAL=$?
-        echo
-        [ $RETVAL -eq 0 ] && touch $lockfile
-        return $RETVAL
-}
-
-stop() {
-        echo -n $"Shutting down $prog: "
-        killproc -p $pidfile $python27
-        RETVAL=$?
-        echo
-        [ $RETVAL -eq 0 ] && rm -f $lockfile
-        return $RETVAL
-}
-
-# See how we were called.
-case "$1" in
-  start)
-        start
-        ;;
-  stop)
-        stop
-        ;;
-  status)
-        status $prog
-        ;;
-  restart|force-reload)
-        stop
-        start
-        ;;
-  try-restart|condrestart)
-        if status $prog > /dev/null; then
-            stop
-            start
-        fi
-        ;;
-  reload)
-        exit 3
-        ;;
-  *)
-        echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}"
-        exit 2
-esac

init-scripts/init.freebsd

@@ -38,6 +38,7 @@ load_rc_config ${name}
 status_cmd="${name}_status"
 stop_cmd="${name}_stop"
 
+command_interpreter="python"
 command="${tautulli_dir}/Tautulli.py"
 command_args="--daemon --pidfile ${tautulli_pid} --quiet --nolaunch ${tautulli_flags}"
 
@@ -51,7 +52,7 @@ verify_tautulli_pid() {
     # Make sure the pid corresponds to the Tautulli process.
     if [ -f ${tautulli_pid} ]; then
       pid=`cat ${tautulli_pid} 2>/dev/null`
-      ps -p ${pid} | grep -q "python2 ${tautulli_dir}/Tautulli.py"
+      ps -p ${pid} | grep -q "python ${tautulli_dir}/Tautulli.py"
       return $?
     else
       return 0
@@ -60,7 +61,7 @@ verify_tautulli_pid() {
 
 # Try to stop Tautulli cleanly by sending SIGTERM
 tautulli_stop() {
-    echo "Stopping $name"
+    echo "Stopping $name."
     verify_tautulli_pid
     if [ -n "${pid}" ]; then
       kill ${pid}

init-scripts/init.freenas

@@ -1,81 +0,0 @@
-#!/bin/sh
-#
-# PROVIDE: tautulli
-# REQUIRE: DAEMON tautulli
-# KEYWORD: shutdown
-#
-# Add the following lines to /etc/rc.conf.local or /etc/rc.conf
-# to enable this service:
-#
-# tautulli_enable (bool): Set to NO by default.
-#           Set it to YES to enable it.
-# tautulli_user:  The user account Tautulli daemon runs as what
-#           you want it to be. It uses 'tautulli' user by
-#           default. Do not sets it as empty or it will run
-#           as root.
-# tautulli_dir:   Directory where Tautulli lives.
-#           Default: /usr/local/share/Tautulli
-# tautulli_chdir:  Change to this directory before running Tautulli.
-#     Default is same as tautulli_dir.
-# tautulli_pid:  The name of the pidfile to create.
-#     Default is tautulli.pid in tautulli_dir.
-PATH="/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
-
-. /etc/rc.subr
-
-name="tautulli"
-rcvar=${name}_enable
-
-load_rc_config ${name}
-
-: ${tautulli_enable:="NO"}
-: ${tautulli_user:="tautulli"}
-: ${tautulli_dir:="/usr/local/share/Tautulli"}
-: ${tautulli_chdir:="${tautulli_dir}"}
-: ${tautulli_pid:="${tautulli_dir}/tautulli.pid"}
-: ${tautulli_flags:=""}
-
-status_cmd="${name}_status"
-stop_cmd="${name}_stop"
-
-command="${tautulli_dir}/Tautulli.py"
-command_args="--daemon --pidfile ${tautulli_pid} --quiet --nolaunch ${tautulli_flags}"
-
-# Ensure user is root when running this script.
-if [ `id -u` != "0" ]; then
-  echo "Oops, you should be root before running this!"
-  exit 1
-fi
-
-verify_tautulli_pid() {
-    # Make sure the pid corresponds to the Tautulli process.
-    if [ -f ${tautulli_pid} ]; then
-      pid=`cat ${tautulli_pid} 2>/dev/null`
-      ps -p ${pid} | grep -q "python2 ${tautulli_dir}/Tautulli.py"
-      return $?
-    else
-      return 0
-    fi
-}
-
-# Try to stop Tautulli cleanly by sending SIGTERM
-tautulli_stop() {
-    echo "Stopping $name."
-    verify_tautulli_pid
-    if [ -n "${pid}" ]; then
-      kill ${pid}
-      wait_for_pids ${pid}
-      echo "Stopped."
-    fi
-}
-
-tautulli_status() {
-    verify_tautulli_pid
-    if [ -n "${pid}" ]; then
-      echo "$name is running as ${pid}."
-    else
-      echo "$name is not running."
-    fi
-}
-
-run_rc_command "$1"

init-scripts/init.freenas

@@ -0,0 +1 @@
+init.freebsd

init-scripts/init.solaris11

@@ -1,47 +0,0 @@
-<?xml version="1.0"?>
-<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
-<!--
-        Created by Manifold
---><service_bundle type="manifest" name="tautulli">
-
-    <service name="application/tautulli" type="service" version="1">
-
-        <create_default_instance enabled="true"/>
-
-        <single_instance/>
-
-        <dependency name="network" grouping="require_all" restart_on="error" type="service">
-            <service_fmri value="svc:/milestone/network:default"/>
-        </dependency>
-
-        <dependency name="filesystem" grouping="require_all" restart_on="error" type="service">
-            <service_fmri value="svc:/system/filesystem/local"/>
-        </dependency>
-
-        <method_context>
-            <method_credential user="tautulli" group="nogroup"/>
-        </method_context>
-
-        <exec_method type="method" name="start" exec="python /opt/Tautulli/Tautulli.py --daemon --quiet --nolaunch" timeout_seconds="60"/>
-
-        <exec_method type="method" name="stop" exec=":kill" timeout_seconds="60"/>
-
-        <property_group name="startd" type="framework">
-            <propval name="duration" type="astring" value="contract"/>
-            <propval name="ignore_error" type="astring" value="core,signal"/>
-        </property_group>
-
-
-        <stability value="Evolving"/>
-
-        <template>
-            <common_name>
-                <loctext xml:lang="C">
-                    Tautulli
-                </loctext>
-            </common_name>
-        </template>
-
-    </service>
-
-</service_bundle>

init-scripts/init.systemd

@@ -28,14 +28,16 @@
 #           Ubuntu/Debian: sudo addgroup tautulli && sudo adduser --system --no-create-home tautulli --ingroup tautulli
 #           CentOS/Fedora: sudo adduser --system --no-create-home tautulli
 #       2. Give the user ownership of the Tautulli directory:
-#           sudo chown tautulli:tautulli -R /opt/Tautulli
+#           sudo chown -R tautulli:tautulli /opt/Tautulli
 #
 #    - Adjust ExecStart= to point to:
-#       1. Your Tautulli executable
+#       1. Your Python interpreter (get the path with "command -v python3")
+#          - Default: /usr/bin/python3
+#       2. Your Tautulli executable
 #          - Default: /opt/Tautulli/Tautulli.py
-#       2. Your config file (recommended is to put it somewhere in /etc)
+#       3. Your config file (recommended is to put it somewhere in /etc)
 #          - Default: --config /opt/Tautulli/config.ini
-#       3. Your datadir (recommended is to NOT put it in your Tautulli exec dir)
+#       4. Your datadir (recommended is to NOT put it in your Tautulli exec dir)
 #          - Default: --datadir /opt/Tautulli
 #
 #    - Adjust User= and Group= to the user/group you want Tautulli to run as.
@@ -50,7 +52,7 @@ Wants=network-online.target
 After=network-online.target
 
 [Service]
-ExecStart=/opt/Tautulli/Tautulli.py --config /opt/Tautulli/config.ini --datadir /opt/Tautulli --quiet --daemon --nolaunch
+ExecStart=/usr/bin/python3 /opt/Tautulli/Tautulli.py --config /opt/Tautulli/config.ini --datadir /opt/Tautulli --quiet --daemon --nolaunch
 GuessMainPID=no
 Type=forking
 User=tautulli

init-scripts/init.ubuntu

@@ -1,209 +0,0 @@
-#!/bin/sh
-#
-## Don't edit this file
-## Edit user configuation in /etc/default/tautulli to change
-##
-## Make sure init script is executable
-## sudo chmod +x /path/to/init.ubuntu
-##
-## Install the init script
-## sudo ln -s /path/to/init.ubuntu /etc/init.d/tautulli
-##
-## Create the tautulli daemon user:
-## sudo adduser --system --no-create-home tautulli
-##
-## Make sure /opt/Tautulli is owned by the tautulli user
-## sudo chown tautulli:nogroup -R /opt/Tautulli
-##
-## Touch the default file to stop the warning message when starting
-## sudo touch /etc/default/tautulli
-##
-## To start Tautulli automatically
-## sudo  update-rc.d tautulli defaults
-##
-## To start/stop/restart Tautulli
-## sudo service tautulli start
-## sudo service tautulli stop
-## sudo service tautulli restart
-##
-## TAUTULLI_USER=         #$RUN_AS, username to run Tautulli under, the default is tautulli
-## TAUTULLI_HOME=         #$APP_PATH, the location of Tautulli.py, the default is /opt/Tautulli
-## TAUTULLI_DATA=         #$DATA_DIR, the location of plexpy.db, cache, logs, the default is /opt/Tautulli
-## TAUTULLI_PIDFILE=      #$PID_FILE, the location of tautulli.pid, the default is /var/run/tautulli/tautulli.pid
-## PYTHON_BIN=      #$DAEMON, the location of the python binary, the default is /usr/bin/python
-## TAUTULLI_OPTS=         #$EXTRA_DAEMON_OPTS, extra cli option for Tautulli, i.e. " --config=/home/Tautulli/config.ini"
-## SSD_OPTS=        #$EXTRA_SSD_OPTS, extra start-stop-daemon option like " --group=users"
-## TAUTULLI_PORT=         #$PORT_OPTS, hardcoded port for the webserver, overrides value in config.ini
-##
-## EXAMPLE if want to run as different user
-## add TAUTULLI_USER=username to /etc/default/tautulli
-## otherwise default tautulli is used
-#
-### BEGIN INIT INFO
-# Provides:          tautulli
-# Required-Start:    $local_fs $network $remote_fs
-# Required-Stop:     $local_fs $network $remote_fs
-# Should-Start:      $NetworkManager
-# Should-Stop:       $NetworkManager
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: starts instance of Tautulli
-# Description:       starts instance of Tautulli using start-stop-daemon
-### END INIT INFO
-
-# Script name
-NAME=tautulli
-
-# App name
-DESC=Tautulli
-
-SETTINGS_LOADED=FALSE
-
-. /lib/lsb/init-functions
-
-# Source Tautulli configuration
-if [ -f /etc/default/tautulli ]; then
-    SETTINGS=/etc/default/tautulli
-else
-    log_warning_msg "/etc/default/tautulli not found using default settings.";
-fi
-
-check_retval() {
-    if [ $? -eq 0 ]; then
-        log_end_msg 0
-        return 0
-    else
-        log_end_msg 1
-        exit 1
-    fi
-}
-
-load_settings() {
-    if [ $SETTINGS_LOADED != "TRUE" ]; then
-        . $SETTINGS
-
-        ## The defaults
-        # Run as username
-        RUN_AS=${TAUTULLI_USER-tautulli}
-
-        # Path to app TAUTULLI_HOME=path_to_app_Tautulli.py
-        APP_PATH=${TAUTULLI_HOME-/opt/Tautulli}
-
-        # Data directory where plexpy.db, cache and logs are stored
-        DATA_DIR=${TAUTULLI_DATA-/opt/Tautulli}
-
-        # Path to store PID file
-        PID_FILE=${TAUTULLI_PIDFILE-/var/run/tautulli/tautulli.pid}
-
-        # Path to python bin
-        DAEMON=${PYTHON_BIN-/usr/bin/python}
-
-        # Extra daemon option like: TAUTULLI_OPTS=" --config=/home/Tautulli/config.ini"
-        EXTRA_DAEMON_OPTS=${TAUTULLI_OPTS-}
-
-        # Extra start-stop-daemon option like START_OPTS=" --group=users"
-        EXTRA_SSD_OPTS=${SSD_OPTS-}
-
-        # Hardcoded port to run on, overrides config.ini settings
-        [ -n "$TAUTULLI_PORT" ] && {
-            PORT_OPTS=" --port=${TAUTULLI_PORT} "
-        }
-
-        DAEMON_OPTS=" Tautulli.py --quiet --daemon --nolaunch --pidfile=${PID_FILE} --datadir=${DATA_DIR} ${PORT_OPTS}${EXTRA_DAEMON_OPTS}"
-
-        SETTINGS_LOADED=TRUE
-    fi
-
-    [ -x $DAEMON ] || {
-        log_warning_msg "$DESC: Can't execute daemon, aborting. See $DAEMON";
-        return 1;}
-
-    return 0
-}
-
-load_settings || exit 0
-
-is_running () {
-    # returns 1 when running, else 0.
-    if [ -e $PID_FILE ]; then
-      PID=`cat $PID_FILE`
-
-      RET=$?
-      [ $RET -gt 1 ] && exit 1 || return $RET
-    else
-      return 1
-    fi
-}
-
-handle_pid () {
-    PID_PATH=`dirname $PID_FILE`
-    [ -d $PID_PATH ] || mkdir -p $PID_PATH && chown -R $RUN_AS $PID_PATH > /dev/null || {
-        log_warning_msg "$DESC: Could not create $PID_FILE, See $SETTINGS, aborting.";
-        return 1;}
-
-    if [ -e $PID_FILE ]; then
-        PID=`cat $PID_FILE`
-        if ! kill -0 $PID > /dev/null 2>&1; then
-            log_warning_msg "Removing stale $PID_FILE"
-            rm $PID_FILE
-        fi
-    fi
-}
-
-handle_datadir () {
-    [ -d $DATA_DIR ] || mkdir -p $DATA_DIR && chown -R $RUN_AS $DATA_DIR > /dev/null || {
-        log_warning_msg "$DESC: Could not create $DATA_DIR, See $SETTINGS, aborting.";
-        return 1;}
-}
-
-handle_updates () {
-    chown -R $RUN_AS $APP_PATH > /dev/null || {
-        log_warning_msg "$DESC: $APP_PATH not writable by $RUN_AS for web-updates";
-        return 0; }
-}
-
-start_tautulli () {
-    handle_pid
-    handle_datadir
-    handle_updates
-    if ! is_running; then
-        log_daemon_msg "Starting $DESC"
-        start-stop-daemon -o -d $APP_PATH -c $RUN_AS --start $EXTRA_SSD_OPTS --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS
-        check_retval
-    else
-        log_success_msg "$DESC: already running (pid $PID)"
-    fi
-}
-
-stop_tautulli () {
-    if is_running; then
-        log_daemon_msg "Stopping $DESC"
-        start-stop-daemon -o --stop --pidfile $PID_FILE --retry 15
-        check_retval
-    else
-        log_success_msg "$DESC: not running"
-    fi
-}
-
-case "$1" in
-    start)
-        start_tautulli
-        ;;
-    stop)
-        stop_tautulli
-        ;;
-    restart|force-reload)
-        stop_tautulli
-        start_tautulli
-        ;;
-    status)
-        status_of_proc -p "$PID_FILE" "$DAEMON" "$DESC"
-        ;;
-    *)
-        N=/etc/init.d/$NAME
-        echo "Usage: $N {start|stop|restart|force-reload|status}" >&2
-        exit 1
-        ;;
-esac
-
-exit 0

init-scripts/init.upstart

@@ -1,18 +0,0 @@
-# tautulli
-#
-# This is a session/user job. Install this file into /usr/share/upstart/sessions
-# if Tautulli is installed system wide, and into $XDG_CONFIG_HOME/upstart if
-# Tautulli is installed per user. Change the executable path appropiately.
-
-start on desktop-start
-stop on desktop-end
-
-env CONFIG=""$XDG_CONFIG_HOME"/Tautulli"
-env DATA=""$XDG_DATA_HOME"/Tautulli"
-
-pre-start script
-    [ -d "$CONFIG" ] || mkdir -p "$CONFIG"
-    [ -d "$DATA" ] || mkdir -p "$DATA"
-end script
-
-exec Tautulli.py --nolaunch --config "$CONFIG"/config.ini --datadir "$DATA"

lib/UniversalAnalytics/HTTPLog.py

@@ -10,7 +10,7 @@
 
 
 import sys, re, os
-from cStringIO import StringIO
+from io import StringIO
 
 
 
@@ -116,6 +116,6 @@ def consume(outbuffer = None): # Capture standard output
 
 if __name__ == '__main__':
     consume(sys.stdout).write(sys.stdin.read())
-    print '\n'
+    print('\n')
 
 # vim: set nowrap tabstop=4 shiftwidth=4 softtabstop=0 expandtab textwidth=0 filetype=python foldmethod=indent foldcolumn=4

lib/UniversalAnalytics/Tracker.py

@@ -1,16 +1,7 @@
-###############################################################################
+from future.moves.urllib.request import urlopen, build_opener, install_opener
-# Universal Analytics for Python
+from future.moves.urllib.request import Request, HTTPSHandler
-# Copyright (c) 2013, Analytics Pros
+from future.moves.urllib.error import URLError, HTTPError
-# 
+from future.moves.urllib.parse import urlencode
-# This project is free software, distributed under the BSD license. 
-# Analytics Pros offers consulting and integration services if your firm needs 
-# assistance in strategy, implementation, or auditing existing work.
-###############################################################################
-
-from urllib2 import urlopen, build_opener, install_opener
-from urllib2 import Request, HTTPSHandler
-from urllib2 import URLError, HTTPError
-from urllib import urlencode
 
 import random
 import datetime
@@ -24,8 +15,8 @@ def generate_uuid(basedata=None):
     """ Provides a _random_ UUID with no input, or a UUID4-format MD5 checksum of any input data provided """
     if basedata is None:
         return str(uuid.uuid4())
-    elif isinstance(basedata, basestring):
+    elif isinstance(basedata, str):
-        checksum = hashlib.md5(basedata).hexdigest()
+        checksum = hashlib.md5(str(basedata).encode('utf-8')).hexdigest()
         return '%8s-%4s-%4s-%4s-%12s' % (
         checksum[0:8], checksum[8:12], checksum[12:16], checksum[16:20], checksum[20:32])
 
@@ -44,7 +35,7 @@ class Time(datetime.datetime):
     def to_unix(cls, timestamp):
         """ Wrapper over time module to produce Unix epoch time as a float """
         if not isinstance(timestamp, datetime.datetime):
-            raise TypeError, 'Time.milliseconds expects a datetime object'
+            raise TypeError('Time.milliseconds expects a datetime object')
         base = time.mktime(timestamp.timetuple())
         return base
 
@@ -86,14 +77,14 @@ class HTTPRequest(object):
     def fixUTF8(cls, data):  # Ensure proper encoding for UA's servers...
         """ Convert all strings to UTF-8 """
         for key in data:
-            if isinstance(data[key], basestring):
+            if isinstance(data[key], str):
                 data[key] = data[key].encode('utf-8')
         return data
 
     # Apply stored properties to the given dataset & POST to the configured endpoint
     def send(self, data):
         request = Request(
-            self.endpoint + '?' + urlencode(self.fixUTF8(data)),
+            self.endpoint + '?' + urlencode(self.fixUTF8(data)).encode('utf-8'),
             headers={
                 'User-Agent': self.user_agent
             }
@@ -121,7 +112,7 @@ class HTTPPost(HTTPRequest):
     def send(self, data):
         request = Request(
             self.endpoint,
-            data=urlencode(self.fixUTF8(data)),
+            data=urlencode(self.fixUTF8(data)).encode('utf-8'),
             headers={
                 'User-Agent': self.user_agent
             }
@@ -144,26 +135,26 @@ class Tracker(object):
 
     @classmethod
     def coerceParameter(cls, name, value=None):
-        if isinstance(name, basestring) and name[0] == '&':
+        if isinstance(name, str) and name[0] == '&':
             return name[1:], str(value)
         elif name in cls.parameter_alias:
             typecast, param_name = cls.parameter_alias.get(name)
             return param_name, typecast(value)
         else:
-            raise KeyError, 'Parameter "{0}" is not recognized'.format(name)
+            raise KeyError('Parameter "{0}" is not recognized'.format(name))
 
     def payload(self, data):
-        for key, value in data.iteritems():
+        for key, value in data.items():
             try:
                 yield self.coerceParameter(key, value)
             except KeyError:
                 continue
 
     option_sequence = {
-        'pageview': [(basestring, 'dp')],
+        'pageview': [(str, 'dp')],
-        'event': [(basestring, 'ec'), (basestring, 'ea'), (basestring, 'el'), (int, 'ev')],
+        'event': [(str, 'ec'), (str, 'ea'), (str, 'el'), (int, 'ev')],
-        'social': [(basestring, 'sn'), (basestring, 'sa'), (basestring, 'st')],
+        'social': [(str, 'sn'), (str, 'sa'), (str, 'st')],
-        'timing': [(basestring, 'utc'), (basestring, 'utv'), (basestring, 'utt'), (basestring, 'utl')]
+        'timing': [(str, 'utc'), (str, 'utv'), (str, 'utt'), (str, 'utl')]
     }
 
     @classmethod
@@ -232,7 +223,7 @@ class Tracker(object):
                 for key, val in self.payload(item):
                     data[key] = val
 
-        for k, v in self.params.iteritems():  # update only absent parameters
+        for k, v in self.params.items():  # update only absent parameters
             if k not in data:
                 data[k] = v
 
@@ -247,13 +238,13 @@ class Tracker(object):
     # Setting persistent attibutes of the session/hit/etc (inc. custom dimensions/metrics)
     def set(self, name, value=None):
         if isinstance(name, dict):
-            for key, value in name.iteritems():
+            for key, value in name.items():
                 try:
                     param, value = self.coerceParameter(key, value)
                     self.params[param] = value
                 except KeyError:
                     pass
-        elif isinstance(name, basestring):
+        elif isinstance(name, str):
             try:
                 param, value = self.coerceParameter(name, value)
                 self.params[param] = value
@@ -277,7 +268,7 @@ class Tracker(object):
 def safe_unicode(obj):
     """ Safe convertion to the Unicode string version of the object """
     try:
-        return unicode(obj)
+        return str(obj)
     except UnicodeDecodeError:
         return obj.decode('utf-8')
 
@@ -380,7 +371,7 @@ for i in range(0, 5):
 # Enhanced Ecommerce
 Tracker.alias(str, 'pa')  # Product action
 Tracker.alias(str, 'tcc')  # Coupon code
-Tracker.alias(unicode, 'pal')  # Product action list
+Tracker.alias(str, 'pal')  # Product action list
 Tracker.alias(int, 'cos')  # Checkout step
 Tracker.alias(str, 'col')  # Checkout step option
 
@@ -388,10 +379,10 @@ Tracker.alias(str, 'promoa')  # Promotion action
 
 for product_index in range(1, MAX_EC_PRODUCTS):
     Tracker.alias(str, 'pr{0}id'.format(product_index))  # Product SKU
-    Tracker.alias(unicode, 'pr{0}nm'.format(product_index))  # Product name
+    Tracker.alias(str, 'pr{0}nm'.format(product_index))  # Product name
-    Tracker.alias(unicode, 'pr{0}br'.format(product_index))  # Product brand
+    Tracker.alias(str, 'pr{0}br'.format(product_index))  # Product brand
-    Tracker.alias(unicode, 'pr{0}ca'.format(product_index))  # Product category
+    Tracker.alias(str, 'pr{0}ca'.format(product_index))  # Product category
-    Tracker.alias(unicode, 'pr{0}va'.format(product_index))  # Product variant
+    Tracker.alias(str, 'pr{0}va'.format(product_index))  # Product variant
     Tracker.alias(str, 'pr{0}pr'.format(product_index))  # Product price
     Tracker.alias(int, 'pr{0}qt'.format(product_index))  # Product quantity
     Tracker.alias(str, 'pr{0}cc'.format(product_index))  # Product coupon code
@@ -403,10 +394,10 @@ for product_index in range(1, MAX_EC_PRODUCTS):
 
     for list_index in range(1, MAX_EC_LISTS):
         Tracker.alias(str, 'il{0}pi{1}id'.format(list_index, product_index))  # Product impression SKU
-        Tracker.alias(unicode, 'il{0}pi{1}nm'.format(list_index, product_index))  # Product impression name
+        Tracker.alias(str, 'il{0}pi{1}nm'.format(list_index, product_index))  # Product impression name
-        Tracker.alias(unicode, 'il{0}pi{1}br'.format(list_index, product_index))  # Product impression brand
+        Tracker.alias(str, 'il{0}pi{1}br'.format(list_index, product_index))  # Product impression brand
-        Tracker.alias(unicode, 'il{0}pi{1}ca'.format(list_index, product_index))  # Product impression category
+        Tracker.alias(str, 'il{0}pi{1}ca'.format(list_index, product_index))  # Product impression category
-        Tracker.alias(unicode, 'il{0}pi{1}va'.format(list_index, product_index))  # Product impression variant
+        Tracker.alias(str, 'il{0}pi{1}va'.format(list_index, product_index))  # Product impression variant
         Tracker.alias(int, 'il{0}pi{1}ps'.format(list_index, product_index))  # Product impression position
         Tracker.alias(int, 'il{0}pi{1}pr'.format(list_index, product_index))  # Product impression price
 
@@ -417,11 +408,11 @@ for product_index in range(1, MAX_EC_PRODUCTS):
                                                         custom_index))  # Product impression custom metric
 
 for list_index in range(1, MAX_EC_LISTS):
-    Tracker.alias(unicode, 'il{0}nm'.format(list_index))  # Product impression list name
+    Tracker.alias(str, 'il{0}nm'.format(list_index))  # Product impression list name
 
 for promotion_index in range(1, MAX_EC_PROMOTIONS):
     Tracker.alias(str, 'promo{0}id'.format(promotion_index))  # Promotion ID
-    Tracker.alias(unicode, 'promo{0}nm'.format(promotion_index))  # Promotion name
+    Tracker.alias(str, 'promo{0}nm'.format(promotion_index))  # Promotion name
     Tracker.alias(str, 'promo{0}cr'.format(promotion_index))  # Promotion creative
     Tracker.alias(str, 'promo{0}ps'.format(promotion_index))  # Promotion position
 

lib/UniversalAnalytics/__init__.py

@@ -1 +1 @@
-import Tracker
+from . import Tracker

lib/appdirs.py

@@ -0,0 +1,608 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2005-2010 ActiveState Software Inc.
+# Copyright (c) 2013 Eddy Petrișor
+
+"""Utilities for determining application-specific dirs.
+
+See <http://github.com/ActiveState/appdirs> for details and usage.
+"""
+# Dev Notes:
+# - MSDN on where to store app data files:
+#   http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120
+# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html
+# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+
+__version_info__ = (1, 4, 3)
+__version__ = '.'.join(map(str, __version_info__))
+
+
+import sys
+import os
+
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    unicode = str
+
+if sys.platform.startswith('java'):
+    import platform
+    os_name = platform.java_ver()[3][0]
+    if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc.
+        system = 'win32'
+    elif os_name.startswith('Mac'): # "Mac OS X", etc.
+        system = 'darwin'
+    else: # "Linux", "SunOS", "FreeBSD", etc.
+        # Setting this to "linux2" is not ideal, but only Windows or Mac
+        # are actually checked for and the rest of the module expects
+        # *sys.platform* style strings.
+        system = 'linux2'
+else:
+    system = sys.platform
+
+
+
+def user_data_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user data directories are:
+        Mac OS X:               ~/Library/Application Support/<AppName>
+        Unix:                   ~/.local/share/<AppName>    # or in $XDG_DATA_HOME, if defined
+        Win XP (not roaming):   C:\Documents and Settings\<username>\Application Data\<AppAuthor>\<AppName>
+        Win XP (roaming):       C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>
+        Win 7  (not roaming):   C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>
+        Win 7  (roaming):       C:\Users\<username>\AppData\Roaming\<AppAuthor>\<AppName>
+
+    For Unix, we follow the XDG spec and support $XDG_DATA_HOME.
+    That means, by default "~/.local/share/<AppName>".
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA"
+        path = os.path.normpath(_get_win_folder(const))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+    elif system == 'darwin':
+        path = os.path.expanduser('~/Library/Application Support/')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def site_data_dir(appname=None, appauthor=None, version=None, multipath=False):
+    r"""Return full path to the user-shared data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "multipath" is an optional parameter only applicable to *nix
+            which indicates that the entire list of data dirs should be
+            returned. By default, the first item from XDG_DATA_DIRS is
+            returned, or '/usr/local/share/<AppName>',
+            if XDG_DATA_DIRS is not set
+
+    Typical site data directories are:
+        Mac OS X:   /Library/Application Support/<AppName>
+        Unix:       /usr/local/share/<AppName> or /usr/share/<AppName>
+        Win XP:     C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName>
+        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
+        Win 7:      C:\ProgramData\<AppAuthor>\<AppName>   # Hidden, but writeable on Win 7.
+
+    For Unix, this is using the $XDG_DATA_DIRS[0] default.
+
+    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA"))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+    elif system == 'darwin':
+        path = os.path.expanduser('/Library/Application Support')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        # XDG default for $XDG_DATA_DIRS
+        # only first, if multipath is False
+        path = os.getenv('XDG_DATA_DIRS',
+                         os.pathsep.join(['/usr/local/share', '/usr/share']))
+        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)]
+        if appname:
+            if version:
+                appname = os.path.join(appname, version)
+            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+
+        if multipath:
+            path = os.pathsep.join(pathlist)
+        else:
+            path = pathlist[0]
+        return path
+
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_config_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific config dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user config directories are:
+        Mac OS X:               same as user_data_dir
+        Unix:                   ~/.config/<AppName>     # or in $XDG_CONFIG_HOME, if defined
+        Win *:                  same as user_data_dir
+
+    For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME.
+    That means, by default "~/.config/<AppName>".
+    """
+    if system in ["win32", "darwin"]:
+        path = user_data_dir(appname, appauthor, None, roaming)
+    else:
+        path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def site_config_dir(appname=None, appauthor=None, version=None, multipath=False):
+    r"""Return full path to the user-shared data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "multipath" is an optional parameter only applicable to *nix
+            which indicates that the entire list of config dirs should be
+            returned. By default, the first item from XDG_CONFIG_DIRS is
+            returned, or '/etc/xdg/<AppName>', if XDG_CONFIG_DIRS is not set
+
+    Typical site config directories are:
+        Mac OS X:   same as site_data_dir
+        Unix:       /etc/xdg/<AppName> or $XDG_CONFIG_DIRS[i]/<AppName> for each value in
+                    $XDG_CONFIG_DIRS
+        Win *:      same as site_data_dir
+        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
+
+    For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False
+
+    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
+    """
+    if system in ["win32", "darwin"]:
+        path = site_data_dir(appname, appauthor)
+        if appname and version:
+            path = os.path.join(path, version)
+    else:
+        # XDG default for $XDG_CONFIG_DIRS
+        # only first, if multipath is False
+        path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg')
+        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)]
+        if appname:
+            if version:
+                appname = os.path.join(appname, version)
+            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+
+        if multipath:
+            path = os.pathsep.join(pathlist)
+        else:
+            path = pathlist[0]
+    return path
+
+
+def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True):
+    r"""Return full path to the user-specific cache dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "opinion" (boolean) can be False to disable the appending of
+            "Cache" to the base app data dir for Windows. See
+            discussion below.
+
+    Typical user cache directories are:
+        Mac OS X:   ~/Library/Caches/<AppName>
+        Unix:       ~/.cache/<AppName> (XDG default)
+        Win XP:     C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Cache
+        Vista:      C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Cache
+
+    On Windows the only suggestion in the MSDN docs is that local settings go in
+    the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming
+    app data dir (the default returned by `user_data_dir` above). Apps typically
+    put cache data somewhere *under* the given dir here. Some examples:
+        ...\Mozilla\Firefox\Profiles\<ProfileName>\Cache
+        ...\Acme\SuperApp\Cache\1.0
+    OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value.
+    This can be disabled with the `opinion=False` option.
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+            if opinion:
+                path = os.path.join(path, "Cache")
+    elif system == 'darwin':
+        path = os.path.expanduser('~/Library/Caches')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_state_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific state dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user state directories are:
+        Mac OS X:  same as user_data_dir
+        Unix:      ~/.local/state/<AppName>   # or in $XDG_STATE_HOME, if defined
+        Win *:     same as user_data_dir
+
+    For Unix, we follow this Debian proposal <https://wiki.debian.org/XDGBaseDirectorySpecification#state>
+    to extend the XDG spec and support $XDG_STATE_HOME.
+
+    That means, by default "~/.local/state/<AppName>".
+    """
+    if system in ["win32", "darwin"]:
+        path = user_data_dir(appname, appauthor, None, roaming)
+    else:
+        path = os.getenv('XDG_STATE_HOME', os.path.expanduser("~/.local/state"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_log_dir(appname=None, appauthor=None, version=None, opinion=True):
+    r"""Return full path to the user-specific log dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "opinion" (boolean) can be False to disable the appending of
+            "Logs" to the base app data dir for Windows, and "log" to the
+            base cache dir for Unix. See discussion below.
+
+    Typical user log directories are:
+        Mac OS X:   ~/Library/Logs/<AppName>
+        Unix:       ~/.cache/<AppName>/log  # or under $XDG_CACHE_HOME if defined
+        Win XP:     C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Logs
+        Vista:      C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Logs
+
+    On Windows the only suggestion in the MSDN docs is that local settings
+    go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in
+    examples of what some windows apps use for a logs dir.)
+
+    OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA`
+    value for Windows and appends "log" to the user cache dir for Unix.
+    This can be disabled with the `opinion=False` option.
+    """
+    if system == "darwin":
+        path = os.path.join(
+            os.path.expanduser('~/Library/Logs'),
+            appname)
+    elif system == "win32":
+        path = user_data_dir(appname, appauthor, version)
+        version = False
+        if opinion:
+            path = os.path.join(path, "Logs")
+    else:
+        path = user_cache_dir(appname, appauthor, version)
+        version = False
+        if opinion:
+            path = os.path.join(path, "log")
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+class AppDirs(object):
+    """Convenience wrapper for getting application dirs."""
+    def __init__(self, appname=None, appauthor=None, version=None,
+            roaming=False, multipath=False):
+        self.appname = appname
+        self.appauthor = appauthor
+        self.version = version
+        self.roaming = roaming
+        self.multipath = multipath
+
+    @property
+    def user_data_dir(self):
+        return user_data_dir(self.appname, self.appauthor,
+                             version=self.version, roaming=self.roaming)
+
+    @property
+    def site_data_dir(self):
+        return site_data_dir(self.appname, self.appauthor,
+                             version=self.version, multipath=self.multipath)
+
+    @property
+    def user_config_dir(self):
+        return user_config_dir(self.appname, self.appauthor,
+                               version=self.version, roaming=self.roaming)
+
+    @property
+    def site_config_dir(self):
+        return site_config_dir(self.appname, self.appauthor,
+                             version=self.version, multipath=self.multipath)
+
+    @property
+    def user_cache_dir(self):
+        return user_cache_dir(self.appname, self.appauthor,
+                              version=self.version)
+
+    @property
+    def user_state_dir(self):
+        return user_state_dir(self.appname, self.appauthor,
+                              version=self.version)
+
+    @property
+    def user_log_dir(self):
+        return user_log_dir(self.appname, self.appauthor,
+                            version=self.version)
+
+
+#---- internal support stuff
+
+def _get_win_folder_from_registry(csidl_name):
+    """This is a fallback technique at best. I'm not sure if using the
+    registry for this guarantees us the correct answer for all CSIDL_*
+    names.
+    """
+    if PY3:
+      import winreg as _winreg
+    else:
+      import _winreg
+
+    shell_folder_name = {
+        "CSIDL_APPDATA": "AppData",
+        "CSIDL_COMMON_APPDATA": "Common AppData",
+        "CSIDL_LOCAL_APPDATA": "Local AppData",
+    }[csidl_name]
+
+    key = _winreg.OpenKey(
+        _winreg.HKEY_CURRENT_USER,
+        r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+    )
+    dir, type = _winreg.QueryValueEx(key, shell_folder_name)
+    return dir
+
+
+def _get_win_folder_with_pywin32(csidl_name):
+    from win32com.shell import shellcon, shell
+    dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0)
+    # Try to make this a unicode path because SHGetFolderPath does
+    # not return unicode strings when there is unicode data in the
+    # path.
+    try:
+        dir = unicode(dir)
+
+        # Downgrade to short path name if have highbit chars. See
+        # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+        has_high_char = False
+        for c in dir:
+            if ord(c) > 255:
+                has_high_char = True
+                break
+        if has_high_char:
+            try:
+                import win32api
+                dir = win32api.GetShortPathName(dir)
+            except ImportError:
+                pass
+    except UnicodeError:
+        pass
+    return dir
+
+
+def _get_win_folder_with_ctypes(csidl_name):
+    import ctypes
+
+    csidl_const = {
+        "CSIDL_APPDATA": 26,
+        "CSIDL_COMMON_APPDATA": 35,
+        "CSIDL_LOCAL_APPDATA": 28,
+    }[csidl_name]
+
+    buf = ctypes.create_unicode_buffer(1024)
+    ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
+
+    # Downgrade to short path name if have highbit chars. See
+    # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+    has_high_char = False
+    for c in buf:
+        if ord(c) > 255:
+            has_high_char = True
+            break
+    if has_high_char:
+        buf2 = ctypes.create_unicode_buffer(1024)
+        if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024):
+            buf = buf2
+
+    return buf.value
+
+def _get_win_folder_with_jna(csidl_name):
+    import array
+    from com.sun import jna
+    from com.sun.jna.platform import win32
+
+    buf_size = win32.WinDef.MAX_PATH * 2
+    buf = array.zeros('c', buf_size)
+    shell = win32.Shell32.INSTANCE
+    shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf)
+    dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+    # Downgrade to short path name if have highbit chars. See
+    # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+    has_high_char = False
+    for c in dir:
+        if ord(c) > 255:
+            has_high_char = True
+            break
+    if has_high_char:
+        buf = array.zeros('c', buf_size)
+        kernel = win32.Kernel32.INSTANCE
+        if kernel.GetShortPathName(dir, buf, buf_size):
+            dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+    return dir
+
+if system == "win32":
+    try:
+        import win32com.shell
+        _get_win_folder = _get_win_folder_with_pywin32
+    except ImportError:
+        try:
+            from ctypes import windll
+            _get_win_folder = _get_win_folder_with_ctypes
+        except ImportError:
+            try:
+                import com.sun.jna
+                _get_win_folder = _get_win_folder_with_jna
+            except ImportError:
+                _get_win_folder = _get_win_folder_from_registry
+
+
+#---- self test code
+
+if __name__ == "__main__":
+    appname = "MyApp"
+    appauthor = "MyCompany"
+
+    props = ("user_data_dir",
+             "user_config_dir",
+             "user_cache_dir",
+             "user_state_dir",
+             "user_log_dir",
+             "site_data_dir",
+             "site_config_dir")
+
+    print("-- app dirs %s --" % __version__)
+
+    print("-- app dirs (with optional 'version')")
+    dirs = AppDirs(appname, appauthor, version="1.0")
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (without optional 'version')")
+    dirs = AppDirs(appname, appauthor)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (without optional 'appauthor')")
+    dirs = AppDirs(appname)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (with disabled 'appauthor')")
+    dirs = AppDirs(appname, appauthor=False)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))

lib/apscheduler/events.py

@@ -3,7 +3,7 @@ __all__ = ('EVENT_SCHEDULER_STARTED', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_SCHEDUL
            'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED',
            'EVENT_JOB_ADDED', 'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED',
            'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED', 'EVENT_JOB_SUBMITTED', 'EVENT_JOB_MAX_INSTANCES',
-           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent')
+           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent', 'JobSubmissionEvent')
 
 
 EVENT_SCHEDULER_STARTED = EVENT_SCHEDULER_START = 2 ** 0

lib/apscheduler/executors/asyncio.py

@@ -3,12 +3,11 @@ from __future__ import absolute_import
 import sys
 
 from apscheduler.executors.base import BaseExecutor, run_job
+from apscheduler.util import iscoroutinefunction_partial
 
 try:
-    from asyncio import iscoroutinefunction
     from apscheduler.executors.base_py3 import run_coroutine_job
 except ImportError:
-    from trollius import iscoroutinefunction
     run_coroutine_job = None
 
 
@@ -46,7 +45,7 @@ class AsyncIOExecutor(BaseExecutor):
             else:
                 self._run_job_success(job.id, events)
 
-        if iscoroutinefunction(job.func):
+        if iscoroutinefunction_partial(job.func):
             if run_coroutine_job is not None:
                 coro = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
                 f = self._eventloop.create_task(coro)

lib/apscheduler/executors/tornado.py

@@ -8,10 +8,10 @@ from tornado.gen import convert_yielded
 from apscheduler.executors.base import BaseExecutor, run_job
 
 try:
-    from inspect import iscoroutinefunction
     from apscheduler.executors.base_py3 import run_coroutine_job
+    from apscheduler.util import iscoroutinefunction_partial
 except ImportError:
-    def iscoroutinefunction(func):
+    def iscoroutinefunction_partial(func):
         return False
 
 
@@ -44,7 +44,7 @@ class TornadoExecutor(BaseExecutor):
             else:
                 self._run_job_success(job.id, events)
 
-        if iscoroutinefunction(job.func):
+        if iscoroutinefunction_partial(job.func):
             f = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
         else:
             f = self.executor.submit(run_job, job, job._jobstore_alias, run_times,

lib/apscheduler/job.py

@@ -1,4 +1,4 @@
-from collections import Iterable, Mapping
+from inspect import ismethod, isclass
 from uuid import uuid4
 
 import six
@@ -8,6 +8,11 @@ from apscheduler.util import (
     ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args,
     convert_to_datetime)
 
+try:
+    from collections.abc import Iterable, Mapping
+except ImportError:
+    from collections import Iterable, Mapping
+
 
 class Job(object):
     """
@@ -235,13 +240,20 @@ class Job(object):
                 'be determined. Consider giving a textual reference (module:function name) '
                 'instead.' % (self.func,))
 
+        # Instance methods cannot survive serialization as-is, so store the "self" argument
+        # explicitly
+        if ismethod(self.func) and not isclass(self.func.__self__):
+            args = (self.func.__self__,) + tuple(self.args)
+        else:
+            args = self.args
+
         return {
             'version': 1,
             'id': self.id,
             'func': self.func_ref,
             'trigger': self.trigger,
             'executor': self.executor,
-            'args': self.args,
+            'args': args,
             'kwargs': self.kwargs,
             'name': self.name,
             'misfire_grace_time': self.misfire_grace_time,

lib/apscheduler/jobstores/redis.py

@@ -14,7 +14,7 @@ except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from redis import StrictRedis
+    from redis import Redis
 except ImportError:  # pragma: nocover
     raise ImportError('RedisJobStore requires redis installed')
 
@@ -47,7 +47,7 @@ class RedisJobStore(BaseJobStore):
         self.pickle_protocol = pickle_protocol
         self.jobs_key = jobs_key
         self.run_times_key = run_times_key
-        self.redis = StrictRedis(db=int(db), **connect_args)
+        self.redis = Redis(db=int(db), **connect_args)
 
     def lookup_job(self, job_id):
         job_state = self.redis.hget(self.jobs_key, job_id)
@@ -81,7 +81,9 @@ class RedisJobStore(BaseJobStore):
             pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
                                                           self.pickle_protocol))
             if job.next_run_time:
-                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
+
             pipe.execute()
 
     def update_job(self, job):
@@ -92,9 +94,11 @@ class RedisJobStore(BaseJobStore):
             pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
                                                           self.pickle_protocol))
             if job.next_run_time:
-                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
             else:
                 pipe.zrem(self.run_times_key, job.id)
+
             pipe.execute()
 
     def remove_job(self, job_id):

lib/apscheduler/jobstores/rethinkdb.py

@@ -10,7 +10,7 @@ except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    import rethinkdb as r
+    from rethinkdb import RethinkDB
 except ImportError:  # pragma: nocover
     raise ImportError('RethinkDBJobStore requires rethinkdb installed')
 
@@ -40,10 +40,12 @@ class RethinkDBJobStore(BaseJobStore):
             raise ValueError('The "table" parameter must not be empty')
 
         self.database = database
-        self.table = table
+        self.table_name = table
+        self.table = None
         self.client = client
         self.pickle_protocol = pickle_protocol
         self.connect_args = connect_args
+        self.r = RethinkDB()
         self.conn = None
 
     def start(self, scheduler, alias):
@@ -52,31 +54,31 @@ class RethinkDBJobStore(BaseJobStore):
         if self.client:
             self.conn = maybe_ref(self.client)
         else:
-            self.conn = r.connect(db=self.database, **self.connect_args)
+            self.conn = self.r.connect(db=self.database, **self.connect_args)
 
-        if self.database not in r.db_list().run(self.conn):
+        if self.database not in self.r.db_list().run(self.conn):
-            r.db_create(self.database).run(self.conn)
+            self.r.db_create(self.database).run(self.conn)
 
-        if self.table not in r.table_list().run(self.conn):
+        if self.table_name not in self.r.table_list().run(self.conn):
-            r.table_create(self.table).run(self.conn)
+            self.r.table_create(self.table_name).run(self.conn)
 
-        if 'next_run_time' not in r.table(self.table).index_list().run(self.conn):
+        if 'next_run_time' not in self.r.table(self.table_name).index_list().run(self.conn):
-            r.table(self.table).index_create('next_run_time').run(self.conn)
+            self.r.table(self.table_name).index_create('next_run_time').run(self.conn)
 
-        self.table = r.db(self.database).table(self.table)
+        self.table = self.r.db(self.database).table(self.table_name)
 
     def lookup_job(self, job_id):
         results = list(self.table.get_all(job_id).pluck('job_state').run(self.conn))
         return self._reconstitute_job(results[0]['job_state']) if results else None
 
     def get_due_jobs(self, now):
-        return self._get_jobs(r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
+        return self._get_jobs(self.r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
 
     def get_next_run_time(self):
         results = list(
             self.table
-            .filter(r.row['next_run_time'] != None)  # flake8: noqa
+            .filter(self.r.row['next_run_time'] != None)  # noqa
-            .order_by(r.asc('next_run_time'))
+            .order_by(self.r.asc('next_run_time'))
             .map(lambda x: x['next_run_time'])
             .limit(1)
             .run(self.conn)
@@ -92,7 +94,7 @@ class RethinkDBJobStore(BaseJobStore):
         job_dict = {
             'id': job.id,
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
-            'job_state': r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
         }
         results = self.table.insert(job_dict).run(self.conn)
         if results['errors'] > 0:
@@ -101,7 +103,7 @@ class RethinkDBJobStore(BaseJobStore):
     def update_job(self, job):
         changes = {
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
-            'job_state': r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
         }
         results = self.table.get_all(job.id).update(changes).run(self.conn)
         skipped = False in map(lambda x: results[x] == 0, results.keys())
@@ -130,20 +132,20 @@ class RethinkDBJobStore(BaseJobStore):
     def _get_jobs(self, predicate=None):
         jobs = []
         failed_job_ids = []
-        query = (self.table.filter(r.row['next_run_time'] != None).filter(predicate) if
+        query = (self.table.filter(self.r.row['next_run_time'] != None).filter(predicate)  # noqa
-                 predicate else self.table)
+                 if predicate else self.table)
         query = query.order_by('next_run_time', 'id').pluck('id', 'job_state')
 
         for document in query.run(self.conn):
             try:
                 jobs.append(self._reconstitute_job(document['job_state']))
-            except:
+            except Exception:
                 self._logger.exception('Unable to restore job "%s" -- removing it', document['id'])
                 failed_job_ids.append(document['id'])
 
         # Remove all the jobs we failed to restore
         if failed_job_ids:
-            r.expr(failed_job_ids).for_each(
+            self.r.expr(failed_job_ids).for_each(
                 lambda job_id: self.table.get_all(job_id).delete()).run(self.conn)
 
         return jobs

lib/apscheduler/jobstores/sqlalchemy.py

@@ -106,7 +106,7 @@ class SQLAlchemyJobStore(BaseJobStore):
         }).where(self.jobs_t.c.id == job.id)
         result = self.engine.execute(update)
         if result.rowcount == 0:
-            raise JobLookupError(id)
+            raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
         delete = self.jobs_t.delete().where(self.jobs_t.c.id == job_id)

lib/apscheduler/schedulers/base.py

@@ -1,7 +1,6 @@
 from __future__ import print_function
 
 from abc import ABCMeta, abstractmethod
-from collections import MutableMapping
 from threading import RLock
 from datetime import datetime, timedelta
 from logging import getLogger
@@ -19,13 +18,19 @@ from apscheduler.jobstores.base import ConflictingIdError, JobLookupError, BaseJ
 from apscheduler.jobstores.memory import MemoryJobStore
 from apscheduler.job import Job
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import asbool, asint, astimezone, maybe_ref, timedelta_seconds, undefined
+from apscheduler.util import (
+    asbool, asint, astimezone, maybe_ref, timedelta_seconds, undefined, TIMEOUT_MAX)
 from apscheduler.events import (
     SchedulerEvent, JobEvent, JobSubmissionEvent, EVENT_SCHEDULER_START, EVENT_SCHEDULER_SHUTDOWN,
     EVENT_JOBSTORE_ADDED, EVENT_JOBSTORE_REMOVED, EVENT_ALL, EVENT_JOB_MODIFIED, EVENT_JOB_REMOVED,
     EVENT_JOB_ADDED, EVENT_EXECUTOR_ADDED, EVENT_EXECUTOR_REMOVED, EVENT_ALL_JOBS_REMOVED,
     EVENT_JOB_SUBMITTED, EVENT_JOB_MAX_INSTANCES, EVENT_SCHEDULER_RESUMED, EVENT_SCHEDULER_PAUSED)
 
+try:
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+
 #: constant indicating a scheduler's stopped state
 STATE_STOPPED = 0
 #: constant indicating a scheduler's running state (started and processing jobs)
@@ -126,11 +131,14 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         :param bool paused: if ``True``, don't start job processing until :meth:`resume` is called
         :raises SchedulerAlreadyRunningError: if the scheduler is already running
+        :raises RuntimeError: if running under uWSGI with threads disabled
 
         """
         if self.state != STATE_STOPPED:
             raise SchedulerAlreadyRunningError
 
+        self._check_uwsgi()
+
         with self._executors_lock:
             # Create a default executor if nothing else is configured
             if 'default' not in self._executors:
@@ -177,12 +185,13 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         self.state = STATE_STOPPED
 
-        with self._jobstores_lock, self._executors_lock:
         # Shut down all executors
+        with self._executors_lock:
             for executor in six.itervalues(self._executors):
                 executor.shutdown(wait)
 
         # Shut down all job stores
+        with self._jobstores_lock:
             for jobstore in six.itervalues(self._jobstores):
                 jobstore.shutdown()
 
@@ -546,7 +555,7 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         """
         if pending is not None:
             warnings.warn('The "pending" option is deprecated -- get_jobs() always returns '
-                          'pending jobs if the scheduler has been started and scheduled jobs '
+                          'scheduled jobs if the scheduler has been started and pending jobs '
                           'otherwise', DeprecationWarning)
 
         with self._jobstores_lock:
@@ -589,7 +598,6 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         """
         jobstore_alias = None
         with self._jobstores_lock:
-            if self.state == STATE_STOPPED:
             # Check if the job is among the pending jobs
             if self.state == STATE_STOPPED:
                 for i, (job, alias, replace_existing) in enumerate(self._pending_jobs):
@@ -824,6 +832,14 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
                 except BaseException:
                     self._logger.exception('Error notifying listener')
 
+    def _check_uwsgi(self):
+        """Check if we're running under uWSGI with threads disabled."""
+        uwsgi_module = sys.modules.get('uwsgi')
+        if not getattr(uwsgi_module, 'has_threads', True):
+            raise RuntimeError('The scheduler seems to be running under uWSGI, but threads have '
+                               'been disabled. You must run uWSGI with the --enable-threads '
+                               'option for the scheduler to work.')
+
     def _real_add_job(self, job, jobstore_alias, replace_existing):
         """
         :param Job job: the job to add
@@ -999,7 +1015,7 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
             wait_seconds = None
             self._logger.debug('No jobs; waiting until a job is added')
         else:
-            wait_seconds = max(timedelta_seconds(next_wakeup_time - now), 0)
+            wait_seconds = min(max(timedelta_seconds(next_wakeup_time - now), 0), TIMEOUT_MAX)
             self._logger.debug('Next wakeup is due at %s (in %f seconds)', next_wakeup_time,
                                wait_seconds)
 

lib/apscheduler/schedulers/qt.py

@@ -9,7 +9,7 @@ except (ImportError, RuntimeError):  # pragma: nocover
         from PyQt4.QtCore import QObject, QTimer
     except ImportError:
         try:
-            from PySide.QtCore import QObject, QTimer  # flake8: noqa
+            from PySide.QtCore import QObject, QTimer  # noqa
         except ImportError:
             raise ImportError('QtScheduler requires either PyQt5, PyQt4 or PySide installed')
 
@@ -26,7 +26,8 @@ class QtScheduler(BaseScheduler):
     def _start_timer(self, wait_seconds):
         self._stop_timer()
         if wait_seconds is not None:
-            self._timer = QTimer.singleShot(wait_seconds * 1000, self._process_jobs)
+            wait_time = min(wait_seconds * 1000, 2147483647)
+            self._timer = QTimer.singleShot(wait_time, self._process_jobs)
 
     def _stop_timer(self):
         if self._timer:

lib/apscheduler/triggers/cron/__init__.py

@@ -192,9 +192,8 @@ class CronTrigger(BaseTrigger):
                 return None
 
         if fieldnum >= 0:
-            if self.jitter is not None:
             next_date = self._apply_jitter(next_date, self.jitter, now)
-            return next_date
+            return min(next_date, self.end_date) if self.end_date else next_date
 
     def __getstate__(self):
         return {

lib/apscheduler/triggers/cron/expressions.py

@@ -9,7 +9,7 @@ __all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression',
            'WeekdayPositionExpression', 'LastDayOfMonthExpression')
 
 
-WEEKDAYS = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']
+WEEKDAYS = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
 MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
 
 

lib/apscheduler/triggers/cron/fields.py

@@ -104,7 +104,7 @@ class DayOfWeekField(BaseField):
     COMPILERS = BaseField.COMPILERS + [WeekdayRangeExpression]
 
     def get_value(self, dateval):
-        return dateval.isoweekday() % 7
+        return dateval.weekday()
 
 
 class MonthField(BaseField):

lib/apscheduler/util.py

@@ -1,12 +1,14 @@
 """This module contains several handy functions primarily meant for internal use."""
 
 from __future__ import division
+
 from datetime import date, datetime, time, timedelta, tzinfo
 from calendar import timegm
-import re
 from functools import partial
+from inspect import isclass, ismethod
+import re
 
-from pytz import timezone, utc
+from pytz import timezone, utc, FixedOffset
 import six
 
 try:
@@ -19,9 +21,19 @@ try:
 except ImportError:
     TIMEOUT_MAX = 4294967  # Maximum value accepted by Event.wait() on Windows
 
+try:
+    from asyncio import iscoroutinefunction
+except ImportError:
+    try:
+        from trollius import iscoroutinefunction
+    except ImportError:
+        def iscoroutinefunction(func):
+            return False
+
 __all__ = ('asint', 'asbool', 'astimezone', 'convert_to_datetime', 'datetime_to_utc_timestamp',
            'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name',
-           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args')
+           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args',
+           'TIMEOUT_MAX')
 
 
 class _Undefined(object):
@@ -92,8 +104,9 @@ def astimezone(obj):
 
 _DATE_REGEX = re.compile(
     r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})'
-    r'(?: (?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
+    r'(?:[ T](?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
-    r'(?:\.(?P<microsecond>\d{1,6}))?)?')
+    r'(?:\.(?P<microsecond>\d{1,6}))?'
+    r'(?P<timezone>Z|[+-]\d\d:\d\d)?)?$')
 
 
 def convert_to_datetime(input, tz, arg_name):
@@ -105,7 +118,9 @@ def convert_to_datetime(input, tz, arg_name):
     If the input is a string, it is parsed as a datetime with the given timezone.
 
     Date strings are accepted in three different forms: date only (Y-m-d), date with time
-    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro).
+    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro). Additionally you can
+    override the time zone by giving a specific offset in the format specified by ISO 8601:
+    Z (UTC), +HH:MM or -HH:MM.
 
     :param str|datetime input: the datetime or string to convert to a timezone aware datetime
     :param datetime.tzinfo tz: timezone to interpret ``input`` in
@@ -123,8 +138,17 @@ def convert_to_datetime(input, tz, arg_name):
         m = _DATE_REGEX.match(input)
         if not m:
             raise ValueError('Invalid date string')
-        values = [(k, int(v or 0)) for k, v in m.groupdict().items()]
+
-        values = dict(values)
+        values = m.groupdict()
+        tzname = values.pop('timezone')
+        if tzname == 'Z':
+            tz = utc
+        elif tzname:
+            hours, minutes = (int(x) for x in tzname[1:].split(':'))
+            sign = 1 if tzname[0] == '+' else -1
+            tz = FixedOffset(sign * (hours * 60 + minutes))
+
+        values = {k: int(v or 0) for k, v in values.items()}
         datetime_ = datetime(**values)
     else:
         raise TypeError('Unsupported type for %s: %s' % (arg_name, input.__class__.__name__))
@@ -210,7 +234,7 @@ def get_callable_name(func):
     # class methods, bound and unbound methods
     f_self = getattr(func, '__self__', None) or getattr(func, 'im_self', None)
     if f_self and hasattr(func, '__name__'):
-        f_class = f_self if isinstance(f_self, type) else f_self.__class__
+        f_class = f_self if isclass(f_self) else f_self.__class__
     else:
         f_class = getattr(func, 'im_class', None)
 
@@ -248,7 +272,18 @@ def obj_to_ref(obj):
     if '<locals>' in name:
         raise ValueError('Cannot create a reference to a nested function')
 
-    return '%s:%s' % (obj.__module__, name)
+    if ismethod(obj):
+        if hasattr(obj, 'im_self') and obj.im_self:
+            # bound method
+            module = obj.im_self.__module__
+        elif hasattr(obj, 'im_class') and obj.im_class:
+            # unbound method
+            module = obj.im_class.__module__
+        else:
+            module = obj.__module__
+    else:
+        module = obj.__module__
+    return '%s:%s' % (module, name)
 
 
 def ref_to_obj(ref):
@@ -383,3 +418,12 @@ def check_callable_args(func, args, kwargs):
         raise ValueError(
             'The target callable does not accept the following keyword arguments: %s' %
             ', '.join(unmatched_kwargs))
+
+
+def iscoroutinefunction_partial(f):
+    while isinstance(f, partial):
+        f = f.func
+
+    # The asyncio version of iscoroutinefunction includes testing for @coroutine
+    # decorations vs. the inspect version which does not.
+    return iscoroutinefunction(f)

lib/argparse.py

@@ -1,4 +1,5 @@
 # Author: Steven J. Bethard <steven.bethard@gmail.com>.
+# Maintainer: Thomas Waldmann <tw@waldmann-edv.de>
 
 """Command-line parsing library
 
@@ -61,7 +62,12 @@ considered public as object names -- the API of the formatter objects is
 still considered an implementation detail.)
 """
 
-__version__ = '1.1'
+__version__ = '1.4.0'  # we use our own version number independant of the
+                       # one in stdlib and we release this on pypi.
+
+__external_lib__ = True  # to make sure the tests really test THIS lib,
+                         # not the builtin one in Python stdlib
+
 __all__ = [
     'ArgumentParser',
     'ArgumentError',
@@ -71,7 +77,6 @@ __all__ = [
     'ArgumentDefaultsHelpFormatter',
     'RawDescriptionHelpFormatter',
     'RawTextHelpFormatter',
-    'MetavarTypeHelpFormatter',
     'Namespace',
     'Action',
     'ONE_OR_MORE',
@@ -83,14 +88,35 @@ __all__ = [
 ]
 
 
-import collections as _collections
 import copy as _copy
 import os as _os
 import re as _re
 import sys as _sys
 import textwrap as _textwrap
 
-from gettext import gettext as _, ngettext
+from gettext import gettext as _
+
+try:
+    set
+except NameError:
+    # for python < 2.4 compatibility (sets module is there since 2.3):
+    from sets import Set as set
+
+try:
+    basestring
+except NameError:
+    basestring = str
+
+try:
+    sorted
+except NameError:
+    # for python < 2.4 compatibility:
+    def sorted(iterable, reverse=False):
+        result = list(iterable)
+        result.sort()
+        if reverse:
+            result.reverse()
+        return result
 
 
 def _callable(obj):
@@ -424,8 +450,7 @@ class HelpFormatter(object):
 
             # produce all arg strings
             elif not action.option_strings:
-                default = self._get_default_metavar_for_positional(action)
+                part = self._format_args(action, action.dest)
-                part = self._format_args(action, default)
 
                 # if it's in a group, strip the outer []
                 if action in group_actions:
@@ -447,7 +472,7 @@ class HelpFormatter(object):
                 # if the Optional takes a value, format is:
                 #    -s ARGS or --long ARGS
                 else:
-                    default = self._get_default_metavar_for_optional(action)
+                    default = action.dest.upper()
                     args_string = self._format_args(action, default)
                     part = '%s %s' % (option_string, args_string)
 
@@ -533,8 +558,7 @@ class HelpFormatter(object):
 
     def _format_action_invocation(self, action):
         if not action.option_strings:
-            default = self._get_default_metavar_for_positional(action)
+            metavar, = self._metavar_formatter(action, action.dest)(1)
-            metavar, = self._metavar_formatter(action, default)(1)
             return metavar
 
         else:
@@ -548,7 +572,7 @@ class HelpFormatter(object):
             # if the Optional takes a value, format is:
             #    -s ARGS, --long ARGS
             else:
-                default = self._get_default_metavar_for_optional(action)
+                default = action.dest.upper()
                 args_string = self._format_args(action, default)
                 for option_string in action.option_strings:
                     parts.append('%s %s' % (option_string, args_string))
@@ -626,12 +650,6 @@ class HelpFormatter(object):
     def _get_help_string(self, action):
         return action.help
 
-    def _get_default_metavar_for_optional(self, action):
-        return action.dest.upper()
-
-    def _get_default_metavar_for_positional(self, action):
-        return action.dest
-
 
 class RawDescriptionHelpFormatter(HelpFormatter):
     """Help message formatter which retains any formatting in descriptions.
@@ -672,22 +690,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
         return help
 
 
-class MetavarTypeHelpFormatter(HelpFormatter):
-    """Help message formatter which uses the argument 'type' as the default
-    metavar value (instead of the argument 'dest')
-
-    Only the name of this class is considered a public API. All the methods
-    provided by the class are considered an implementation detail.
-    """
-
-    def _get_default_metavar_for_optional(self, action):
-        return action.type.__name__
-
-    def _get_default_metavar_for_positional(self, action):
-        return action.type.__name__
-
-
-
 # =====================
 # Options and Arguments
 # =====================
@@ -1067,7 +1069,7 @@ class _SubParsersAction(Action):
 
         self._prog_prefix = prog
         self._parser_class = parser_class
-        self._name_parser_map = _collections.OrderedDict()
+        self._name_parser_map = {}
         self._choices_actions = []
 
         super(_SubParsersAction, self).__init__(
@@ -1116,9 +1118,8 @@ class _SubParsersAction(Action):
         try:
             parser = self._name_parser_map[parser_name]
         except KeyError:
-            args = {'parser_name': parser_name,
+            tup = parser_name, ', '.join(self._name_parser_map)
-                    'choices': ', '.join(self._name_parser_map)}
+            msg = _('unknown parser %r (choices: %s)' % tup)
-            msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args
             raise ArgumentError(self, msg)
 
         # parse all the remaining options into the namespace
@@ -1147,7 +1148,7 @@ class FileType(object):
             the builtin open() function.
     """
 
-    def __init__(self, mode='r', bufsize=-1):
+    def __init__(self, mode='r', bufsize=None):
         self._mode = mode
         self._bufsize = bufsize
 
@@ -1159,19 +1160,23 @@ class FileType(object):
             elif 'w' in self._mode:
                 return _sys.stdout
             else:
-                msg = _('argument "-" with mode %r') % self._mode
+                msg = _('argument "-" with mode %r' % self._mode)
                 raise ValueError(msg)
 
-        # all other arguments are used as file names
         try:
+            # all other arguments are used as file names
+            if self._bufsize:
                 return open(string, self._mode, self._bufsize)
-        except IOError as e:
+            else:
+                return open(string, self._mode)
+        except IOError:
+            err = _sys.exc_info()[1]
             message = _("can't open '%s': %s")
-            raise ArgumentTypeError(message % (string, e))
+            raise ArgumentTypeError(message % (string, err))
 
     def __repr__(self):
-        args = self._mode, self._bufsize
+        args = [self._mode, self._bufsize]
-        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
+        args_str = ', '.join([repr(arg) for arg in args if arg is not None])
         return '%s(%s)' % (type(self).__name__, args_str)
 
 # ===========================
@@ -1189,6 +1194,8 @@ class Namespace(_AttributeHolder):
         for name in kwargs:
             setattr(self, name, kwargs[name])
 
+    __hash__ = None
+
     def __eq__(self, other):
         return vars(self) == vars(other)
 
@@ -1312,20 +1319,13 @@ class _ActionsContainer(object):
         # create the action object, and add it to the parser
         action_class = self._pop_action_class(kwargs)
         if not _callable(action_class):
-            raise ValueError('unknown action "%s"' % (action_class,))
+            raise ValueError('unknown action "%s"' % action_class)
         action = action_class(**kwargs)
 
         # raise an error if the action type is not callable
         type_func = self._registry_get('type', action.type, action.type)
         if not _callable(type_func):
-            raise ValueError('%r is not callable' % (type_func,))
+            raise ValueError('%r is not callable' % type_func)
-
-        # raise an error if the metavar does not match the type
-        if hasattr(self, "_get_formatter"):
-            try:
-                self._get_formatter()._format_args(action, None)
-            except TypeError:
-                raise ValueError("length of metavar tuple does not match nargs")
 
         return self._add_action(action)
 
@@ -1426,11 +1426,10 @@ class _ActionsContainer(object):
         for option_string in args:
             # error on strings that don't start with an appropriate prefix
             if not option_string[0] in self.prefix_chars:
-                args = {'option': option_string,
+                msg = _('invalid option string %r: '
-                        'prefix_chars': self.prefix_chars}
+                        'must start with a character %r')
-                msg = _('invalid option string %(option)r: '
+                tup = option_string, self.prefix_chars
-                        'must start with a character %(prefix_chars)r')
+                raise ValueError(msg % tup)
-                raise ValueError(msg % args)
 
             # strings starting with two prefix characters are long options
             option_strings.append(option_string)
@@ -1483,9 +1482,7 @@ class _ActionsContainer(object):
             conflict_handler(action, confl_optionals)
 
     def _handle_conflict_error(self, action, conflicting_actions):
-        message = ngettext('conflicting option string: %s',
+        message = _('conflicting option string(s): %s')
-                           'conflicting option strings: %s',
-                           len(conflicting_actions))
         conflict_string = ', '.join([option_string
                                      for option_string, action
                                      in conflicting_actions])
@@ -1528,7 +1525,6 @@ class _ArgumentGroup(_ActionsContainer):
         self._defaults = container._defaults
         self._has_negative_number_optionals = \
             container._has_negative_number_optionals
-        self._mutually_exclusive_groups = container._mutually_exclusive_groups
 
     def _add_action(self, action):
         action = super(_ArgumentGroup, self)._add_action(action)
@@ -1630,7 +1626,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
 
         # add help and version arguments if necessary
         # (using explicit default to override global argument_default)
-        default_prefix = '-' if '-' in prefix_chars else prefix_chars[0]
+        if '-' in prefix_chars:
+            default_prefix = '-'
+        else:
+            default_prefix = prefix_chars[0]
         if self.add_help:
             self.add_argument(
                 default_prefix+'h', default_prefix*2+'help',
@@ -1743,10 +1742,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
             if action.dest is not SUPPRESS:
                 if not hasattr(namespace, action.dest):
                     if action.default is not SUPPRESS:
-                        default = action.default
+                        setattr(namespace, action.dest, action.default)
-                        if isinstance(action.default, str):
-                            default = self._get_value(action, default)
-                        setattr(namespace, action.dest, default)
 
         # add any parser defaults that aren't present
         for dest in self._defaults:
@@ -1969,12 +1965,28 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         # if we didn't consume all the argument strings, there were extras
         extras.extend(arg_strings[stop_index:])
 
-        # make sure all required actions were present
+        # if we didn't use all the Positional objects, there were too few
-        required_actions = [_get_action_name(action) for action in self._actions
+        # arg strings supplied.
-                            if action.required and action not in seen_actions]
+        if positionals:
-        if required_actions:
+            self.error(_('too few arguments'))
-            self.error(_('the following arguments are required: %s') %
+
-                       ', '.join(required_actions))
+        # make sure all required actions were present, and convert defaults.
+        for action in self._actions:
+            if action not in seen_actions:
+                if action.required:
+                    name = _get_action_name(action)
+                    self.error(_('argument %s is required') % name)
+                else:
+                    # Convert action default now instead of doing it before
+                    # parsing arguments to avoid calling convert functions
+                    # twice (which may fail) if the argument was given, but
+                    # only if it was defined already in the namespace
+                    if (action.default is not None and
+                            isinstance(action.default, basestring) and
+                            hasattr(namespace, action.dest) and
+                            action.default is getattr(namespace, action.dest)):
+                        setattr(namespace, action.dest,
+                                self._get_value(action, action.default))
 
         # make sure all required groups had one option present
         for group in self._mutually_exclusive_groups:
@@ -2038,9 +2050,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 OPTIONAL: _('expected at most one argument'),
                 ONE_OR_MORE: _('expected at least one argument'),
             }
-            default = ngettext('expected %s argument',
+            default = _('expected %s argument(s)') % action.nargs
-                               'expected %s arguments',
-                               action.nargs) % action.nargs
             msg = nargs_errors.get(action.nargs, default)
             raise ArgumentError(action, msg)
 
@@ -2096,9 +2106,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         if len(option_tuples) > 1:
             options = ', '.join([option_string
                 for action, option_string, explicit_arg in option_tuples])
-            args = {'option': arg_string, 'matches': options}
+            tup = arg_string, options
-            msg = _('ambiguous option: %(option)s could match %(matches)s')
+            self.error(_('ambiguous option: %s could match %s') % tup)
-            self.error(msg % args)
 
         # if exactly one action matched, this segmentation is good,
         # so return the parsed action
@@ -2220,7 +2229,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 value = action.const
             else:
                 value = action.default
-            if isinstance(value, str):
+            if isinstance(value, basestring):
                 value = self._get_value(action, value)
                 self._check_value(action, value)
 
@@ -2277,9 +2286,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         # TypeErrors or ValueErrors also indicate errors
         except (TypeError, ValueError):
             name = getattr(action.type, '__name__', repr(action.type))
-            args = {'type': name, 'value': arg_string}
+            msg = _('invalid %s value: %r')
-            msg = _('invalid %(type)s value: %(value)r')
+            raise ArgumentError(action, msg % (name, arg_string))
-            raise ArgumentError(action, msg % args)
 
         # return the converted value
         return result
@@ -2287,10 +2295,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
     def _check_value(self, action, value):
         # converted value must be one of the choices (if specified)
         if action.choices is not None and value not in action.choices:
-            args = {'value': value,
+            tup = value, ', '.join(map(repr, action.choices))
-                    'choices': ', '.join(map(repr, action.choices))}
+            msg = _('invalid choice: %r (choose from %s)') % tup
-            msg = _('invalid choice: %(value)r (choose from %(choices)s)')
+            raise ArgumentError(action, msg)
-            raise ArgumentError(action, msg % args)
 
     # =======================
     # Help-formatting methods
@@ -2382,5 +2389,4 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         should either exit or raise an exception.
         """
         self.print_usage(_sys.stderr)
-        args = {'prog': self.prog, 'message': message}
+        self.exit(2, _('%s: error: %s\n') % (self.prog, message))
-        self.exit(2, _('%(prog)s: error: %(message)s\n') % args)

lib/backports/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

lib/backports/functools_lru_cache.py

@@ -0,0 +1,196 @@
+from __future__ import absolute_import
+
+import functools
+from collections import namedtuple
+from threading import RLock
+
+_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
+
+
+@functools.wraps(functools.update_wrapper)
+def update_wrapper(
+    wrapper,
+    wrapped,
+    assigned=functools.WRAPPER_ASSIGNMENTS,
+    updated=functools.WRAPPER_UPDATES,
+):
+    """
+    Patch two bugs in functools.update_wrapper.
+    """
+    # workaround for http://bugs.python.org/issue3445
+    assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr))
+    wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated)
+    # workaround for https://bugs.python.org/issue17482
+    wrapper.__wrapped__ = wrapped
+    return wrapper
+
+
+class _HashedSeq(list):
+    __slots__ = 'hashvalue'
+
+    def __init__(self, tup, hash=hash):
+        self[:] = tup
+        self.hashvalue = hash(tup)
+
+    def __hash__(self):
+        return self.hashvalue
+
+
+def _make_key(
+    args,
+    kwds,
+    typed,
+    kwd_mark=(object(),),
+    fasttypes=set([int, str, frozenset, type(None)]),
+    sorted=sorted,
+    tuple=tuple,
+    type=type,
+    len=len,
+):
+    'Make a cache key from optionally typed positional and keyword arguments'
+    key = args
+    if kwds:
+        sorted_items = sorted(kwds.items())
+        key += kwd_mark
+        for item in sorted_items:
+            key += item
+    if typed:
+        key += tuple(type(v) for v in args)
+        if kwds:
+            key += tuple(type(v) for k, v in sorted_items)
+    elif len(key) == 1 and type(key[0]) in fasttypes:
+        return key[0]
+    return _HashedSeq(key)
+
+
+def lru_cache(maxsize=100, typed=False):
+    """Least-recently-used cache decorator.
+
+    If *maxsize* is set to None, the LRU features are disabled and the cache
+    can grow without bound.
+
+    If *typed* is True, arguments of different types will be cached separately.
+    For example, f(3.0) and f(3) will be treated as distinct calls with
+    distinct results.
+
+    Arguments to the cached function must be hashable.
+
+    View the cache statistics named tuple (hits, misses, maxsize, currsize) with
+    f.cache_info().  Clear the cache and statistics with f.cache_clear().
+    Access the underlying function with f.__wrapped__.
+
+    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
+
+    """
+
+    # Users should only access the lru_cache through its public API:
+    #       cache_info, cache_clear, and f.__wrapped__
+    # The internals of the lru_cache are encapsulated for thread safety and
+    # to allow the implementation to change (including a possible C version).
+
+    def decorating_function(user_function):
+
+        cache = dict()
+        stats = [0, 0]  # make statistics updateable non-locally
+        HITS, MISSES = 0, 1  # names for the stats fields
+        make_key = _make_key
+        cache_get = cache.get  # bound method to lookup key or return None
+        _len = len  # localize the global len() function
+        lock = RLock()  # because linkedlist updates aren't threadsafe
+        root = []  # root of the circular doubly linked list
+        root[:] = [root, root, None, None]  # initialize by pointing to self
+        nonlocal_root = [root]  # make updateable non-locally
+        PREV, NEXT, KEY, RESULT = 0, 1, 2, 3  # names for the link fields
+
+        if maxsize == 0:
+
+            def wrapper(*args, **kwds):
+                # no caching, just do a statistics update after a successful call
+                result = user_function(*args, **kwds)
+                stats[MISSES] += 1
+                return result
+
+        elif maxsize is None:
+
+            def wrapper(*args, **kwds):
+                # simple caching without ordering or size limit
+                key = make_key(args, kwds, typed)
+                result = cache_get(
+                    key, root
+                )  # root used here as a unique not-found sentinel
+                if result is not root:
+                    stats[HITS] += 1
+                    return result
+                result = user_function(*args, **kwds)
+                cache[key] = result
+                stats[MISSES] += 1
+                return result
+
+        else:
+
+            def wrapper(*args, **kwds):
+                # size limited caching that tracks accesses by recency
+                key = make_key(args, kwds, typed) if kwds or typed else args
+                with lock:
+                    link = cache_get(key)
+                    if link is not None:
+                        # record recent use of the key by moving it
+                        # to the front of the list
+                        root, = nonlocal_root
+                        link_prev, link_next, key, result = link
+                        link_prev[NEXT] = link_next
+                        link_next[PREV] = link_prev
+                        last = root[PREV]
+                        last[NEXT] = root[PREV] = link
+                        link[PREV] = last
+                        link[NEXT] = root
+                        stats[HITS] += 1
+                        return result
+                result = user_function(*args, **kwds)
+                with lock:
+                    root, = nonlocal_root
+                    if key in cache:
+                        # getting here means that this same key was added to the
+                        # cache while the lock was released.  since the link
+                        # update is already done, we need only return the
+                        # computed result and update the count of misses.
+                        pass
+                    elif _len(cache) >= maxsize:
+                        # use the old root to store the new key and result
+                        oldroot = root
+                        oldroot[KEY] = key
+                        oldroot[RESULT] = result
+                        # empty the oldest link and make it the new root
+                        root = nonlocal_root[0] = oldroot[NEXT]
+                        oldkey = root[KEY]
+                        root[KEY] = root[RESULT] = None
+                        # now update the cache dictionary for the new links
+                        del cache[oldkey]
+                        cache[key] = oldroot
+                    else:
+                        # put result in a new link at the front of the list
+                        last = root[PREV]
+                        link = [last, root, key, result]
+                        last[NEXT] = root[PREV] = cache[key] = link
+                    stats[MISSES] += 1
+                return result
+
+        def cache_info():
+            """Report cache statistics"""
+            with lock:
+                return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache))
+
+        def cache_clear():
+            """Clear the cache and cache statistics"""
+            with lock:
+                cache.clear()
+                root = nonlocal_root[0]
+                root[:] = [root, root, None, None]
+                stats[:] = [0, 0]
+
+        wrapper.__wrapped__ = user_function
+        wrapper.cache_info = cache_info
+        wrapper.cache_clear = cache_clear
+        return update_wrapper(wrapper, user_function)
+
+    return decorating_function

lib/bs4/__init__.py

@@ -5,26 +5,30 @@ http://www.crummy.com/software/BeautifulSoup/
 
 Beautiful Soup uses a pluggable XML or HTML parser to parse a
 (possibly invalid) document into a tree representation. Beautiful Soup
-provides provides methods and Pythonic idioms that make it easy to
+provides methods and Pythonic idioms that make it easy to navigate,
-navigate, search, and modify the parse tree.
+search, and modify the parse tree.
 
-Beautiful Soup works with Python 2.6 and up. It works better if lxml
+Beautiful Soup works with Python 2.7 and up. It works better if lxml
 and/or html5lib is installed.
 
 For more than you ever wanted to know about Beautiful Soup, see the
 documentation:
 http://www.crummy.com/software/BeautifulSoup/bs4/doc/
+
 """
 
 __author__ = "Leonard Richardson (leonardr@segfault.org)"
-__version__ = "4.3.2"
+__version__ = "4.8.1"
-__copyright__ = "Copyright (c) 2004-2013 Leonard Richardson"
+__copyright__ = "Copyright (c) 2004-2019 Leonard Richardson"
+# Use of this source code is governed by the MIT license.
 __license__ = "MIT"
 
 __all__ = ['BeautifulSoup']
 
 import os
 import re
+import sys
+import traceback
 import warnings
 
 from .builder import builder_registry, ParserRejectedMarkup
@@ -45,7 +49,7 @@ from .element import (
 
 # The very first thing we do is give a useful error if someone is
 # running this code under Python 3 without converting it.
-syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
+'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'!='You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
 
 class BeautifulSoup(Tag):
     """
@@ -59,7 +63,7 @@ class BeautifulSoup(Tag):
       handle_starttag(name, attrs) # See note about return value
       handle_endtag(name)
       handle_data(data) # Appends to the current data node
-      endData(containerClass=NavigableString) # Ends the current data node
+      endData(containerClass) # Ends the current data node
 
     No matter how complicated the underlying parser is, you should be
     able to build a tree using 'start tag' events, 'end tag' events,
@@ -69,7 +73,7 @@ class BeautifulSoup(Tag):
     like HTML's <br> tag), call handle_starttag and then
     handle_endtag.
     """
-    ROOT_TAG_NAME = u'[document]'
+    ROOT_TAG_NAME = '[document]'
 
     # If the end-user gives no indication which tree builder they
     # want, look for one with these features.
@@ -77,13 +81,62 @@ class BeautifulSoup(Tag):
    
     ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
 
+    NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features=\"%(parser)s\"' to the BeautifulSoup constructor.\n"
+
     def __init__(self, markup="", features=None, builder=None,
-                 parse_only=None, from_encoding=None, **kwargs):
+                 parse_only=None, from_encoding=None, exclude_encodings=None,
-        """The Soup object is initialized as the 'root tag', and the
+                 element_classes=None, **kwargs):
-        provided markup (which can be a string or a file-like object)
+        """Constructor.
-        is fed into the underlying parser."""
+
+        :param markup: A string or a file-like object representing
+        markup to be parsed.
+
+        :param features: Desirable features of the parser to be used. This
+        may be the name of a specific parser ("lxml", "lxml-xml",
+        "html.parser", or "html5lib") or it may be the type of markup
+        to be used ("html", "html5", "xml"). It's recommended that you
+        name a specific parser, so that Beautiful Soup gives you the
+        same results across platforms and virtual environments.
+
+        :param builder: A TreeBuilder subclass to instantiate (or
+        instance to use) instead of looking one up based on
+        `features`. You only need to use this if you've implemented a
+        custom TreeBuilder.
+
+        :param parse_only: A SoupStrainer. Only parts of the document
+        matching the SoupStrainer will be considered. This is useful
+        when parsing part of a document that would otherwise be too
+        large to fit into memory.
+
+        :param from_encoding: A string indicating the encoding of the
+        document to be parsed. Pass this in if Beautiful Soup is
+        guessing wrongly about the document's encoding.
+
+        :param exclude_encodings: A list of strings indicating
+        encodings known to be wrong. Pass this in if you don't know
+        the document's encoding but you know Beautiful Soup's guess is
+        wrong.
+
+        :param element_classes: A dictionary mapping BeautifulSoup
+        classes like Tag and NavigableString to other classes you'd
+        like to be instantiated instead as the parse tree is
+        built. This is useful for using subclasses to modify the
+        default behavior of Tag or NavigableString.
+
+        :param kwargs: For backwards compatibility purposes, the
+        constructor accepts certain keyword arguments used in
+        Beautiful Soup 3. None of these arguments do anything in
+        Beautiful Soup 4; they will result in a warning and then be ignored.
+
+        Apart from this, any keyword arguments passed into the BeautifulSoup
+        constructor are propagated to the TreeBuilder constructor. This
+        makes it possible to configure a TreeBuilder beyond saying
+        which one to use.
+
+        """
 
         if 'convertEntities' in kwargs:
+            del kwargs['convertEntities']
             warnings.warn(
                 "BS4 does not respect the convertEntities argument to the "
                 "BeautifulSoup constructor. Entities are always converted "
@@ -114,9 +167,9 @@ class BeautifulSoup(Tag):
             del kwargs['isHTML']
             warnings.warn(
                 "BS4 does not respect the isHTML argument to the "
-                "BeautifulSoup constructor. You can pass in features='html' "
+                "BeautifulSoup constructor. Suggest you use "
-                "or features='xml' to get a builder capable of handling "
+                "features='lxml' for HTML and features='lxml-xml' for "
-                "one or the other.")
+                "XML.")
 
         def deprecated_argument(old_name, new_name):
             if old_name in kwargs:
@@ -134,13 +187,24 @@ class BeautifulSoup(Tag):
         from_encoding = from_encoding or deprecated_argument(
             "fromEncoding", "from_encoding")
 
-        if len(kwargs) > 0:
+        if from_encoding and isinstance(markup, str):
-            arg = kwargs.keys().pop()
+            warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.")
-            raise TypeError(
+            from_encoding = None
-                "__init__() got an unexpected keyword argument '%s'" % arg)
 
-        if builder is None:
+        self.element_classes = element_classes or dict()
-            if isinstance(features, basestring):
+
+        # We need this information to track whether or not the builder
+        # was specified well enough that we can omit the 'you need to
+        # specify a parser' warning.
+        original_builder = builder
+        original_features = features
+            
+        if isinstance(builder, type):
+            # A builder class was passed in; it needs to be instantiated.
+            builder_class = builder
+            builder = None
+        elif builder is None:
+            if isinstance(features, str):
                 features = [features]
             if features is None or len(features) == 0:
                 features = self.DEFAULT_BUILDER_FEATURES
@@ -150,21 +214,73 @@ class BeautifulSoup(Tag):
                     "Couldn't find a tree builder with the features you "
                     "requested: %s. Do you need to install a parser library?"
                     % ",".join(features))
-            builder = builder_class()
+
+        # At this point either we have a TreeBuilder instance in
+        # builder, or we have a builder_class that we can instantiate
+        # with the remaining **kwargs.
+        if builder is None:
+            builder = builder_class(**kwargs)
+            if not original_builder and not (
+                    original_features == builder.NAME or
+                    original_features in builder.ALTERNATE_NAMES
+            ):
+                if builder.is_xml:
+                    markup_type = "XML"
+                else:
+                    markup_type = "HTML"
+
+                # This code adapted from warnings.py so that we get the same line
+                # of code as our warnings.warn() call gets, even if the answer is wrong
+                # (as it may be in a multithreading situation).
+                caller = None
+                try:
+                    caller = sys._getframe(1)
+                except ValueError:
+                    pass
+                if caller:
+                    globals = caller.f_globals
+                    line_number = caller.f_lineno
+                else:
+                    globals = sys.__dict__
+                    line_number= 1                    
+                filename = globals.get('__file__')
+                if filename:
+                    fnl = filename.lower()
+                    if fnl.endswith((".pyc", ".pyo")):
+                        filename = filename[:-1]
+                if filename:
+                    # If there is no filename at all, the user is most likely in a REPL,
+                    # and the warning is not necessary.
+                    values = dict(
+                        filename=filename,
+                        line_number=line_number,
+                        parser=builder.NAME,
+                        markup_type=markup_type
+                    )
+                    warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % values, stacklevel=2)
+        else:
+            if kwargs:
+                warnings.warn("Keyword arguments to the BeautifulSoup constructor will be ignored. These would normally be passed into the TreeBuilder constructor, but a TreeBuilder instance was passed in as `builder`.")
+                    
         self.builder = builder
         self.is_xml = builder.is_xml
-        self.builder.soup = self
+        self.known_xml = self.is_xml
-
+        self._namespaces = dict()
         self.parse_only = parse_only
 
+        self.builder.initialize_soup(self)
+
         if hasattr(markup, 'read'):        # It's a file-type object.
             markup = markup.read()
-        elif len(markup) <= 256:
+        elif len(markup) <= 256 and (
+                (isinstance(markup, bytes) and not b'<' in markup)
+                or (isinstance(markup, str) and not '<' in markup)
+        ):
             # Print out warnings for a couple beginner problems
             # involving passing non-markup to Beautiful Soup.
             # Beautiful Soup will still parse the input as markup,
             # just in case that's what the user really wants.
-            if (isinstance(markup, unicode)
+            if (isinstance(markup, str)
                 and not os.path.supports_unicode_filenames):
                 possible_filename = markup.encode("utf8")
             else:
@@ -172,37 +288,93 @@ class BeautifulSoup(Tag):
             is_file = False
             try:
                 is_file = os.path.exists(possible_filename)
-            except Exception, e:
+            except Exception as e:
                 # This is almost certainly a problem involving
                 # characters not valid in filenames on this
                 # system. Just let it go.
                 pass
             if is_file:
+                if isinstance(markup, str):
+                    markup = markup.encode("utf8")
                 warnings.warn(
-                    '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup)
+                    '"%s" looks like a filename, not markup. You should'
-            if markup[:5] == "http:" or markup[:6] == "https:":
+                    ' probably open this file and pass the filehandle into'
-                # TODO: This is ugly but I couldn't get it to work in
+                    ' Beautiful Soup.' % markup)
-                # Python 3 otherwise.
+            self._check_markup_is_url(markup)
-                if ((isinstance(markup, bytes) and not b' ' in markup)
-                    or (isinstance(markup, unicode) and not u' ' in markup)):
-                    warnings.warn(
-                        '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)
 
+        rejections = []
+        success = False
         for (self.markup, self.original_encoding, self.declared_html_encoding,
          self.contains_replacement_characters) in (
-            self.builder.prepare_markup(markup, from_encoding)):
+             self.builder.prepare_markup(
+                 markup, from_encoding, exclude_encodings=exclude_encodings)):
             self.reset()
             try:
                 self._feed()
+                success = True
                 break
-            except ParserRejectedMarkup:
+            except ParserRejectedMarkup as e:
+                rejections.append(e)
                 pass
 
+        if not success:
+            other_exceptions = [str(e) for e in rejections]
+            raise ParserRejectedMarkup(
+                "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.\n\nOriginal exception(s) from parser:\n " + "\n ".join(other_exceptions)
+            )
+
         # Clear out the markup and remove the builder's circular
         # reference to this object.
         self.markup = None
         self.builder.soup = None
 
+    def __copy__(self):
+        copy = type(self)(
+            self.encode('utf-8'), builder=self.builder, from_encoding='utf-8'
+        )
+
+        # Although we encoded the tree to UTF-8, that may not have
+        # been the encoding of the original markup. Set the copy's
+        # .original_encoding to reflect the original object's
+        # .original_encoding.
+        copy.original_encoding = self.original_encoding
+        return copy
+
+    def __getstate__(self):
+        # Frequently a tree builder can't be pickled.
+        d = dict(self.__dict__)
+        if 'builder' in d and not self.builder.picklable:
+            d['builder'] = None
+        return d
+
+    @staticmethod
+    def _check_markup_is_url(markup):
+        """ 
+        Check if markup looks like it's actually a url and raise a warning 
+        if so. Markup can be unicode or str (py2) / bytes (py3).
+        """
+        if isinstance(markup, bytes):
+            space = b' '
+            cant_start_with = (b"http:", b"https:")
+        elif isinstance(markup, str):
+            space = ' '
+            cant_start_with = ("http:", "https:")
+        else:
+            return
+
+        if any(markup.startswith(prefix) for prefix in cant_start_with):
+            if not space in markup:
+                if isinstance(markup, bytes):
+                    decoded_markup = markup.decode('utf-8', 'replace')
+                else:
+                    decoded_markup = markup
+                warnings.warn(
+                    '"%s" looks like a URL. Beautiful Soup is not an'
+                    ' HTTP client. You should probably use an HTTP client like'
+                    ' requests to get the document behind the URL, and feed'
+                    ' that document to Beautiful Soup.' % decoded_markup
+                )
+
     def _feed(self):
         # Convert the document to Unicode.
         self.builder.reset()
@@ -223,15 +395,21 @@ class BeautifulSoup(Tag):
         self.preserve_whitespace_tag_stack = []
         self.pushTag(self)
 
-    def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
+    def new_tag(self, name, namespace=None, nsprefix=None, attrs={},
+                sourceline=None, sourcepos=None, **kwattrs):
         """Create a new tag associated with this soup."""
-        return Tag(None, self.builder, name, namespace, nsprefix, attrs)
+        kwattrs.update(attrs)
+        return self.element_classes.get(Tag, Tag)(
+            None, self.builder, name, namespace, nsprefix, kwattrs,
+            sourceline=sourceline, sourcepos=sourcepos
+        )
 
-    def new_string(self, s, subclass=NavigableString):
+    def new_string(self, s, subclass=None):
         """Create a new NavigableString associated with this soup."""
-        navigable = subclass(s)
+        subclass = subclass or self.element_classes.get(
-        navigable.setup()
+            NavigableString, NavigableString
-        return navigable
+        )
+        return subclass(s)
 
     def insert_before(self, successor):
         raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
@@ -250,16 +428,26 @@ class BeautifulSoup(Tag):
 
     def pushTag(self, tag):
         #print "Push", tag.name
-        if self.currentTag:
+        if self.currentTag is not None:
             self.currentTag.contents.append(tag)
         self.tagStack.append(tag)
         self.currentTag = self.tagStack[-1]
         if tag.name in self.builder.preserve_whitespace_tags:
             self.preserve_whitespace_tag_stack.append(tag)
 
-    def endData(self, containerClass=NavigableString):
+    def endData(self, containerClass=None):
+
+        # Default container is NavigableString.
+        containerClass = containerClass or NavigableString
+
+        # The user may want us to instantiate some alias for the
+        # container class.
+        containerClass = self.element_classes.get(
+            containerClass, containerClass
+        )
+        
         if self.current_data:
-            current_data = u''.join(self.current_data)
+            current_data = ''.join(self.current_data)
             # If whitespace is not preserved, and this string contains
             # nothing but ASCII spaces, replace it with a single space
             # or newline.
@@ -289,15 +477,72 @@ class BeautifulSoup(Tag):
 
     def object_was_parsed(self, o, parent=None, most_recent_element=None):
         """Add an object to the parse tree."""
-        parent = parent or self.currentTag
+        if parent is None:
-        most_recent_element = most_recent_element or self._most_recent_element
+            parent = self.currentTag
-        o.setup(parent, most_recent_element)
-
         if most_recent_element is not None:
-            most_recent_element.next_element = o
+            previous_element = most_recent_element
+        else:
+            previous_element = self._most_recent_element
+
+        next_element = previous_sibling = next_sibling = None
+        if isinstance(o, Tag):
+            next_element = o.next_element
+            next_sibling = o.next_sibling
+            previous_sibling = o.previous_sibling
+            if previous_element is None:
+                previous_element = o.previous_element
+
+        fix = parent.next_element is not None
+
+        o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)
+
         self._most_recent_element = o
         parent.contents.append(o)
 
+        # Check if we are inserting into an already parsed node.
+        if fix:
+            self._linkage_fixer(parent)
+
+    def _linkage_fixer(self, el):
+        """Make sure linkage of this fragment is sound."""
+
+        first = el.contents[0]
+        child = el.contents[-1]
+        descendant = child
+
+        if child is first and el.parent is not None:
+            # Parent should be linked to first child
+            el.next_element = child
+            # We are no longer linked to whatever this element is
+            prev_el = child.previous_element
+            if prev_el is not None and prev_el is not el:
+                prev_el.next_element = None
+            # First child should be linked to the parent, and no previous siblings.
+            child.previous_element = el
+            child.previous_sibling = None
+
+        # We have no sibling as we've been appended as the last.
+        child.next_sibling = None
+
+        # This index is a tag, dig deeper for a "last descendant"
+        if isinstance(child, Tag) and child.contents:
+            descendant = child._last_descendant(False)
+
+        # As the final step, link last descendant. It should be linked
+        # to the parent's next sibling (if found), else walk up the chain
+        # and find a parent with a sibling. It should have no next sibling.
+        descendant.next_element = None
+        descendant.next_sibling = None
+        target = el
+        while True:
+            if target is None:
+                break
+            elif target.next_sibling is not None:
+                descendant.next_element = target.next_sibling
+                target.next_sibling.previous_element = child
+                break
+            target = target.parent
+
     def _popToTag(self, name, nsprefix=None, inclusivePop=True):
         """Pops the tag stack up to and including the most recent
         instance of the given tag. If inclusivePop is false, pops the tag
@@ -321,11 +566,12 @@ class BeautifulSoup(Tag):
 
         return most_recently_popped
 
-    def handle_starttag(self, name, namespace, nsprefix, attrs):
+    def handle_starttag(self, name, namespace, nsprefix, attrs, sourceline=None,
+                        sourcepos=None):
         """Push a start tag on to the stack.
 
         If this method returns None, the tag was rejected by the
-        SoupStrainer. You should proceed as if the tag had not occured
+        SoupStrainer. You should proceed as if the tag had not occurred
         in the document. For instance, if this was a self-closing tag,
         don't call handle_endtag.
         """
@@ -338,11 +584,14 @@ class BeautifulSoup(Tag):
                  or not self.parse_only.search_tag(name, attrs))):
             return None
 
-        tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
+        tag = self.element_classes.get(Tag, Tag)(
-                  self.currentTag, self._most_recent_element)
+            self, self.builder, name, namespace, nsprefix, attrs,
+            self.currentTag, self._most_recent_element,
+            sourceline=sourceline, sourcepos=sourcepos
+        )
         if tag is None:
             return tag
-        if self._most_recent_element:
+        if self._most_recent_element is not None:
             self._most_recent_element.next_element = tag
         self._most_recent_element = tag
         self.pushTag(tag)
@@ -367,9 +616,9 @@ class BeautifulSoup(Tag):
             encoding_part = ''
             if eventual_encoding != None:
                 encoding_part = ' encoding="%s"' % eventual_encoding
-            prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
+            prefix = '<?xml version="1.0"%s?>\n' % encoding_part
         else:
-            prefix = u''
+            prefix = ''
         if not pretty_print:
             indent_level = None
         else:
@@ -403,4 +652,4 @@ class FeatureNotFound(ValueError):
 if __name__ == '__main__':
     import sys
     soup = BeautifulSoup(sys.stdin)
-    print soup.prettify()
+    print(soup.prettify())

lib/bs4/builder/__init__.py

@@ -1,10 +1,13 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 from collections import defaultdict
 import itertools
 import sys
 from bs4.element import (
     CharsetMetaAttributeValue,
     ContentMetaAttributeValue,
-    whitespace_re
+    nonwhitespace_re
     )
 
 __all__ = [
@@ -80,20 +83,69 @@ builder_registry = TreeBuilderRegistry()
 class TreeBuilder(object):
     """Turn a document into a Beautiful Soup object tree."""
 
+    NAME = "[Unknown tree builder]"
+    ALTERNATE_NAMES = []
     features = []
 
     is_xml = False
-    preserve_whitespace_tags = set()
+    picklable = False
     empty_element_tags = None # A tag will be considered an empty-element
                               # tag when and only when it has no contents.
     
     # A value for these tag/attribute combinations is a space- or
     # comma-separated list of CDATA, rather than a single CDATA.
-    cdata_list_attributes = {}
+    DEFAULT_CDATA_LIST_ATTRIBUTES = {}
 
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set()
     
-    def __init__(self):
+    USE_DEFAULT = object()
+
+    # Most parsers don't keep track of line numbers.
+    TRACKS_LINE_NUMBERS = False
+    
+    def __init__(self, multi_valued_attributes=USE_DEFAULT,
+                 preserve_whitespace_tags=USE_DEFAULT,
+                 store_line_numbers=USE_DEFAULT):
+        """Constructor.
+
+        :param multi_valued_attributes: If this is set to None, the
+        TreeBuilder will not turn any values for attributes like
+        'class' into lists. Setting this do a dictionary will
+        customize this behavior; look at DEFAULT_CDATA_LIST_ATTRIBUTES
+        for an example.
+
+        Internally, these are called "CDATA list attributes", but that
+        probably doesn't make sense to an end-user, so the argument name
+        is `multi_valued_attributes`.
+
+        :param preserve_whitespace_tags: A list of tags to treat
+        the way <pre> tags are treated in HTML. Tags in this list
+        will have 
+
+        :param store_line_numbers: If the parser keeps track of the
+        line numbers and positions of the original markup, that
+        information will, by default, be stored in each corresponding
+        `Tag` object. You can turn this off by passing
+        store_line_numbers=False. If the parser you're using doesn't 
+        keep track of this information, then setting store_line_numbers=True
+        will do nothing.
+        """
         self.soup = None
+        if multi_valued_attributes is self.USE_DEFAULT:
+            multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES
+        self.cdata_list_attributes = multi_valued_attributes
+        if preserve_whitespace_tags is self.USE_DEFAULT:
+            preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS
+        self.preserve_whitespace_tags = preserve_whitespace_tags
+        if store_line_numbers == self.USE_DEFAULT:
+            store_line_numbers = self.TRACKS_LINE_NUMBERS
+        self.store_line_numbers = store_line_numbers
+        
+    def initialize_soup(self, soup):
+        """The BeautifulSoup object has been initialized and is now
+        being associated with the TreeBuilder.
+        """
+        self.soup = soup
         
     def reset(self):
         pass
@@ -123,8 +175,8 @@ class TreeBuilder(object):
         raise NotImplementedError()
 
     def prepare_markup(self, markup, user_specified_encoding=None,
-                       document_declared_encoding=None):
+                       document_declared_encoding=None, exclude_encodings=None):
-        return markup, None, None, False
+        yield markup, None, None, False
 
     def test_fragment_to_document(self, fragment):
         """Wrap an HTML fragment to make it look like a document.
@@ -153,14 +205,14 @@ class TreeBuilder(object):
             universal = self.cdata_list_attributes.get('*', [])
             tag_specific = self.cdata_list_attributes.get(
                 tag_name.lower(), None)
-            for attr in attrs.keys():
+            for attr in list(attrs.keys()):
                 if attr in universal or (tag_specific and attr in tag_specific):
                     # We have a "class"-type attribute whose string
                     # value is a whitespace-separated list of
                     # values. Split it into a list.
                     value = attrs[attr]
-                    if isinstance(value, basestring):
+                    if isinstance(value, str):
-                        values = whitespace_re.split(value)
+                        values = nonwhitespace_re.findall(value)
                     else:
                         # html5lib sometimes calls setAttributes twice
                         # for the same tag when rearranging the parse
@@ -224,9 +276,19 @@ class HTMLTreeBuilder(TreeBuilder):
     Such as which tags are empty-element tags.
     """
 
-    preserve_whitespace_tags = set(['pre', 'textarea'])
+    empty_element_tags = set([
-    empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
+        # These are from HTML5.
-                              'spacer', 'link', 'frame', 'base'])
+        'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
+        
+        # These are from earlier versions of HTML and are removed in HTML5.
+        'basefont', 'bgsound', 'command', 'frame', 'image', 'isindex', 'nextid', 'spacer'
+    ])
+
+    # The HTML standard defines these as block-level elements. Beautiful
+    # Soup does not treat these elements differently from other elements,
+    # but it may do so eventually, and this information is available if
+    # you need to use it.
+    block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"])
     
     # The HTML standard defines these attributes as containing a
     # space-separated list of values, not a single value. That is,
@@ -235,7 +297,7 @@ class HTMLTreeBuilder(TreeBuilder):
     # encounter one of these attributes, we will parse its value into
     # a list of values if possible. Upon output, the list will be
     # converted back into a string.
-    cdata_list_attributes = {
+    DEFAULT_CDATA_LIST_ATTRIBUTES = {
         "*" : ['class', 'accesskey', 'dropzone'],
         "a" : ['rel', 'rev'],
         "link" :  ['rel', 'rev'],
@@ -252,6 +314,8 @@ class HTMLTreeBuilder(TreeBuilder):
         "output" : ["for"],
         }
 
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
+    
     def set_up_substitutions(self, tag):
         # We are only interested in <meta> tags
         if tag.name != 'meta':
@@ -299,7 +363,14 @@ def register_treebuilders_from(module):
             this_module.builder_registry.register(obj)
 
 class ParserRejectedMarkup(Exception):
-    pass
+    def __init__(self, message_or_exception):
+        """Explain why the parser rejected the given markup, either
+        with a textual explanation or another exception.
+        """
+        if isinstance(message_or_exception, Exception):
+            e = message_or_exception
+            message_or_exception = "%s: %s" % (e.__class__.__name__, str(e))
+        super(ParserRejectedMarkup, self).__init__(message_or_exception)
             
 # Builders are registered in reverse order of priority, so that custom
 # builder registrations will take precedence. In general, we want lxml

lib/bs4/builder/_html5lib.py

@@ -1,17 +1,27 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'HTML5TreeBuilder',
     ]
 
 import warnings
+import re
 from bs4.builder import (
     PERMISSIVE,
     HTML,
     HTML_5,
     HTMLTreeBuilder,
     )
-from bs4.element import NamespacedAttribute
+from bs4.element import (
+    NamespacedAttribute,
+    nonwhitespace_re,
+)
 import html5lib
-from html5lib.constants import namespaces
+from html5lib.constants import (
+    namespaces,
+    prefixes,
+    )
 from bs4.element import (
     Comment,
     Doctype,
@@ -19,14 +29,36 @@ from bs4.element import (
     Tag,
     )
 
+try:
+    # Pre-0.99999999
+    from html5lib.treebuilders import _base as treebuilder_base
+    new_html5lib = False
+except ImportError as e:
+    # 0.99999999 and up
+    from html5lib.treebuilders import base as treebuilder_base
+    new_html5lib = True
+
 class HTML5TreeBuilder(HTMLTreeBuilder):
     """Use html5lib to build a tree."""
 
-    features = ['html5lib', PERMISSIVE, HTML_5, HTML]
+    NAME = "html5lib"
 
-    def prepare_markup(self, markup, user_specified_encoding):
+    features = [NAME, PERMISSIVE, HTML_5, HTML]
+
+    # html5lib can tell us which line number and position in the
+    # original file is the source of an element.
+    TRACKS_LINE_NUMBERS = True
+    
+    def prepare_markup(self, markup, user_specified_encoding,
+                       document_declared_encoding=None, exclude_encodings=None):
         # Store the user-specified encoding for use later on.
         self.user_specified_encoding = user_specified_encoding
+
+        # document_declared_encoding and exclude_encodings aren't used
+        # ATM because the html5lib TreeBuilder doesn't use
+        # UnicodeDammit.
+        if exclude_encodings:
+            warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
         yield (markup, None, None, False)
 
     # These methods are defined by Beautiful Soup.
@@ -34,32 +66,63 @@ class HTML5TreeBuilder(HTMLTreeBuilder):
         if self.soup.parse_only is not None:
             warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
         parser = html5lib.HTMLParser(tree=self.create_treebuilder)
-        doc = parser.parse(markup, encoding=self.user_specified_encoding)
+        self.underlying_builder.parser = parser
+        extra_kwargs = dict()
+        if not isinstance(markup, str):
+            if new_html5lib:
+                extra_kwargs['override_encoding'] = self.user_specified_encoding
+            else:
+                extra_kwargs['encoding'] = self.user_specified_encoding
+        doc = parser.parse(markup, **extra_kwargs)
         
         # Set the character encoding detected by the tokenizer.
-        if isinstance(markup, unicode):
+        if isinstance(markup, str):
             # We need to special-case this because html5lib sets
             # charEncoding to UTF-8 if it gets Unicode input.
             doc.original_encoding = None
         else:
-            doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
+            original_encoding = parser.tokenizer.stream.charEncoding[0]
+            if not isinstance(original_encoding, str):
+                # In 0.99999999 and up, the encoding is an html5lib
+                # Encoding object. We want to use a string for compatibility
+                # with other tree builders.
+                original_encoding = original_encoding.name
+            doc.original_encoding = original_encoding
+        self.underlying_builder.parser = None
             
     def create_treebuilder(self, namespaceHTMLElements):
         self.underlying_builder = TreeBuilderForHtml5lib(
-            self.soup, namespaceHTMLElements)
+            namespaceHTMLElements, self.soup,
+            store_line_numbers=self.store_line_numbers
+        )
         return self.underlying_builder
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""
-        return u'<html><head></head><body>%s</body></html>' % fragment
+        return '<html><head></head><body>%s</body></html>' % fragment
 
 
-class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
+class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
     
-    def __init__(self, soup, namespaceHTMLElements):
+    def __init__(self, namespaceHTMLElements, soup=None,
+                 store_line_numbers=True, **kwargs):
+        if soup:
             self.soup = soup
+        else:
+            from bs4 import BeautifulSoup
+            # TODO: Why is the parser 'html.parser' here? To avoid an
+            # infinite loop?
+            self.soup = BeautifulSoup(
+                "", "html.parser", store_line_numbers=store_line_numbers,
+                **kwargs
+            )
         super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
 
+        # This will be set later to an html5lib.html5parser.HTMLParser
+        # object, which we can use to track the current line number.
+        self.parser = None
+        self.store_line_numbers = store_line_numbers
+        
     def documentClass(self):
         self.soup.reset()
         return Element(self.soup, self.soup, None)
@@ -73,14 +136,26 @@ class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
         self.soup.object_was_parsed(doctype)
 
     def elementClass(self, name, namespace):
-        tag = self.soup.new_tag(name, namespace)
+        kwargs = {}
+        if self.parser and self.store_line_numbers:
+            # This represents the point immediately after the end of the
+            # tag. We don't know when the tag started, but we do know
+            # where it ended -- the character just before this one.
+            sourceline, sourcepos = self.parser.tokenizer.stream.position()
+            kwargs['sourceline'] = sourceline
+            kwargs['sourcepos'] = sourcepos-1
+        tag = self.soup.new_tag(name, namespace, **kwargs)
+
         return Element(tag, self.soup, namespace)
 
     def commentClass(self, data):
         return TextNode(Comment(data), self.soup)
 
     def fragmentClass(self):
-        self.soup = BeautifulSoup("")
+        from bs4 import BeautifulSoup
+        # TODO: Why is the parser 'html.parser' here? To avoid an
+        # infinite loop?
+        self.soup = BeautifulSoup("", "html.parser")
         self.soup.name = "[document_fragment]"
         return Element(self.soup, self.soup, None)
 
@@ -92,7 +167,57 @@ class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
         return self.soup
 
     def getFragment(self):
-        return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
+        return treebuilder_base.TreeBuilder.getFragment(self).element
+
+    def testSerializer(self, element):
+        from bs4 import BeautifulSoup
+        rv = []
+        doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$')
+
+        def serializeElement(element, indent=0):
+            if isinstance(element, BeautifulSoup):
+                pass
+            if isinstance(element, Doctype):
+                m = doctype_re.match(element)
+                if m:
+                    name = m.group(1)
+                    if m.lastindex > 1:
+                        publicId = m.group(2) or ""
+                        systemId = m.group(3) or m.group(4) or ""
+                        rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (' ' * indent, name, publicId, systemId))
+                    else:
+                        rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name))
+                else:
+                    rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
+            elif isinstance(element, Comment):
+                rv.append("|%s<!-- %s -->" % (' ' * indent, element))
+            elif isinstance(element, NavigableString):
+                rv.append("|%s\"%s\"" % (' ' * indent, element))
+            else:
+                if element.namespace:
+                    name = "%s %s" % (prefixes[element.namespace],
+                                      element.name)
+                else:
+                    name = element.name
+                rv.append("|%s<%s>" % (' ' * indent, name))
+                if element.attrs:
+                    attributes = []
+                    for name, value in list(element.attrs.items()):
+                        if isinstance(name, NamespacedAttribute):
+                            name = "%s %s" % (prefixes[name.namespace], name.name)
+                        if isinstance(value, list):
+                            value = " ".join(value)
+                        attributes.append((name, value))
+
+                    for name, value in sorted(attributes):
+                        rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
+                indent += 2
+                for child in element.children:
+                    serializeElement(child, indent)
+        serializeElement(element, 0)
+
+        return "\n".join(rv)
 
 class AttrList(object):
     def __init__(self, element):
@@ -101,7 +226,16 @@ class AttrList(object):
     def __iter__(self):
         return list(self.attrs.items()).__iter__()
     def __setitem__(self, name, value):
-        "set attr", name, value
+        # If this attribute is a multi-valued attribute for this element,
+        # turn its value into a list.
+        list_attr = self.element.cdata_list_attributes
+        if (name in list_attr['*']
+            or (self.element.name in list_attr
+                and name in list_attr[self.element.name])):
+            # A node that is being cloned may have already undergone
+            # this procedure.
+            if not isinstance(value, list):
+                value = nonwhitespace_re.findall(value)
         self.element[name] = value
     def items(self):
         return list(self.attrs.items())
@@ -115,16 +249,16 @@ class AttrList(object):
         return name in list(self.attrs.keys())
 
 
-class Element(html5lib.treebuilders._base.Node):
+class Element(treebuilder_base.Node):
     def __init__(self, element, soup, namespace):
-        html5lib.treebuilders._base.Node.__init__(self, element.name)
+        treebuilder_base.Node.__init__(self, element.name)
         self.element = element
         self.soup = soup
         self.namespace = namespace
 
     def appendChild(self, node):
         string_child = child = None
-        if isinstance(node, basestring):
+        if isinstance(node, str):
             # Some other piece of code decided to pass in a string
             # instead of creating a TextElement object to contain the
             # string.
@@ -136,13 +270,15 @@ class Element(html5lib.treebuilders._base.Node):
             child = node
         elif node.element.__class__ == NavigableString:
             string_child = child = node.element
+            node.parent = self
         else:
             child = node.element
+            node.parent = self
 
-        if not isinstance(child, basestring) and child.parent is not None:
+        if not isinstance(child, str) and child.parent is not None:
             node.element.extract()
 
-        if (string_child and self.element.contents
+        if (string_child is not None and self.element.contents
             and self.element.contents[-1].__class__ == NavigableString):
             # We are appending a string onto another string.
             # TODO This has O(n^2) performance, for input like
@@ -152,7 +288,7 @@ class Element(html5lib.treebuilders._base.Node):
             old_element.replace_with(new_element)
             self.soup._most_recent_element = new_element
         else:
-            if isinstance(node, basestring):
+            if isinstance(node, str):
                 # Create a brand new NavigableString from this string.
                 child = self.soup.new_string(node)
 
@@ -161,6 +297,12 @@ class Element(html5lib.treebuilders._base.Node):
             # immediately after the parent, if it has no children.)
             if self.element.contents:
                 most_recent_element = self.element._last_descendant(False)
+            elif self.element.next_element is not None:
+                # Something from further ahead in the parse tree is
+                # being inserted into this earlier element. This is
+                # very annoying because it means an expensive search
+                # for the last element in the tree.
+                most_recent_element = self.soup._last_descendant()
             else:
                 most_recent_element = self.element
 
@@ -169,9 +311,12 @@ class Element(html5lib.treebuilders._base.Node):
                 most_recent_element=most_recent_element)
 
     def getAttributes(self):
+        if isinstance(self.element, Comment):
+            return {}
         return AttrList(self.element)
 
     def setAttributes(self, attributes):
+
         if attributes is not None and len(attributes) > 0:
 
             converted_attributes = []
@@ -183,7 +328,7 @@ class Element(html5lib.treebuilders._base.Node):
 
             self.soup.builder._replace_cdata_list_attribute_values(
                 self.name, attributes)
-            for name, value in attributes.items():
+            for name, value in list(attributes.items()):
                 self.element[name] = value
 
             # The attributes may contain variables that need substitution.
@@ -195,11 +340,11 @@ class Element(html5lib.treebuilders._base.Node):
     attributes = property(getAttributes, setAttributes)
 
     def insertText(self, data, insertBefore=None):
-        if insertBefore:
         text = TextNode(self.soup.new_string(data), self.soup)
-            self.insertBefore(data, insertBefore)
+        if insertBefore:
+            self.insertBefore(text, insertBefore)
         else:
-            self.appendChild(data)
+            self.appendChild(text)
 
     def insertBefore(self, node, refNode):
         index = self.element.index(refNode.element)
@@ -218,6 +363,10 @@ class Element(html5lib.treebuilders._base.Node):
 
     def reparentChildren(self, new_parent):
         """Move all of this tag's children into another tag."""
+        # print "MOVE", self.element.contents
+        # print "FROM", self.element
+        # print "TO", new_parent.element
+
         element = self.element
         new_parent_element = new_parent.element
         # Determine what this tag's next_element will be once all the children
@@ -236,18 +385,35 @@ class Element(html5lib.treebuilders._base.Node):
             new_parents_last_descendant_next_element = new_parent_element.next_element
 
         to_append = element.contents
-        append_after = new_parent.element.contents
         if len(to_append) > 0:
             # Set the first child's previous_element and previous_sibling
             # to elements within the new parent
             first_child = to_append[0]
+            if new_parents_last_descendant is not None:
                 first_child.previous_element = new_parents_last_descendant
+            else:
+                first_child.previous_element = new_parent_element
             first_child.previous_sibling = new_parents_last_child
+            if new_parents_last_descendant is not None:
+                new_parents_last_descendant.next_element = first_child
+            else:
+                new_parent_element.next_element = first_child
+            if new_parents_last_child is not None:
+                new_parents_last_child.next_sibling = first_child
 
-            # Fix the last child's next_element and next_sibling
+            # Find the very last element being moved. It is now the
-            last_child = to_append[-1]
+            # parent's last descendant. It has no .next_sibling and
-            last_child.next_element = new_parents_last_descendant_next_element
+            # its .next_element is whatever the previous last
-            last_child.next_sibling = None
+            # descendant had.
+            last_childs_last_descendant = to_append[-1]._last_descendant(False, True)
+
+            last_childs_last_descendant.next_element = new_parents_last_descendant_next_element
+            if new_parents_last_descendant_next_element is not None:
+                # TODO: This code has no test coverage and I'm not sure
+                # how to get html5lib to go through this path, but it's
+                # just the other side of the previous line.
+                new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant
+            last_childs_last_descendant.next_sibling = None
 
         for child in to_append:
             child.parent = new_parent_element
@@ -257,6 +423,10 @@ class Element(html5lib.treebuilders._base.Node):
         element.contents = []
         element.next_element = final_next_element
 
+        # print "DONE WITH MOVE"
+        # print "FROM", self.element
+        # print "TO", new_parent_element
+
     def cloneNode(self):
         tag = self.soup.new_tag(self.element.name, self.namespace)
         node = Element(tag, self.soup, self.namespace)
@@ -268,7 +438,7 @@ class Element(html5lib.treebuilders._base.Node):
         return self.element.contents
 
     def getNameTuple(self):
-        if self.namespace is None:
+        if self.namespace == None:
             return namespaces["html"], self.name
         else:
             return self.namespace, self.name
@@ -277,7 +447,7 @@ class Element(html5lib.treebuilders._base.Node):
 
 class TextNode(Element):
     def __init__(self, element, soup):
-        html5lib.treebuilders._base.Node.__init__(self, None)
+        treebuilder_base.Node.__init__(self, None)
         self.element = element
         self.soup = soup
 

lib/bs4/builder/_htmlparser.py

@@ -1,13 +1,23 @@
+# encoding: utf-8
 """Use the HTMLParser library to parse HTML files that aren't too bad."""
 
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'HTMLParserTreeBuilder',
     ]
 
-from HTMLParser import (
+from future.moves.html.parser import HTMLParser
-    HTMLParser,
+
-    HTMLParseError,
+try:
-    )
+    from html.parser import HTMLParseError
+except ImportError as e:
+    # HTMLParseError is removed in Python 3.5. Since it can never be
+    # thrown in 3.5, we can just define our own class as a placeholder.
+    class HTMLParseError(Exception):
+        pass
+
 import sys
 import warnings
 
@@ -19,10 +29,10 @@ import warnings
 # At the end of this file, we monkeypatch HTMLParser so that
 # strict=True works well on Python 3.2.2.
 major, minor, release = sys.version_info[:3]
-CONSTRUCTOR_TAKES_STRICT = (
+CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3
-    major > 3
+CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3
-    or (major == 3 and minor > 2)
+CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4
-    or (major == 3 and minor == 2 and release >= 3))
+
 
 from bs4.element import (
     CData,
@@ -43,7 +53,42 @@ from bs4.builder import (
 HTMLPARSER = 'html.parser'
 
 class BeautifulSoupHTMLParser(HTMLParser):
-    def handle_starttag(self, name, attrs):
+
+    def __init__(self, *args, **kwargs):
+        HTMLParser.__init__(self, *args, **kwargs)
+
+        # Keep a list of empty-element tags that were encountered
+        # without an explicit closing tag. If we encounter a closing tag
+        # of this type, we'll associate it with one of those entries.
+        #
+        # This isn't a stack because we don't care about the
+        # order. It's a list of closing tags we've already handled and
+        # will ignore, assuming they ever show up.
+        self.already_closed_empty_element = []
+
+    def error(self, msg):
+        """In Python 3, HTMLParser subclasses must implement error(), although this
+        requirement doesn't appear to be documented.
+
+        In Python 2, HTMLParser implements error() as raising an exception.
+
+        In any event, this method is called only on very strange markup and our best strategy
+        is to pretend it didn't happen and keep going.
+        """
+        warnings.warn(msg)
+        
+    def handle_startendtag(self, name, attrs):
+        # This is only called when the markup looks like
+        # <tag/>.
+
+        # is_startend() tells handle_starttag not to close the tag
+        # just because its name matches a known empty-element tag. We
+        # know that this is an empty-element tag and we want to call
+        # handle_endtag ourselves.
+        tag = self.handle_starttag(name, attrs, handle_empty_element=False)
+        self.handle_endtag(name)
+        
+    def handle_starttag(self, name, attrs, handle_empty_element=True):
         # XXX namespace
         attr_dict = {}
         for key, value in attrs:
@@ -53,9 +98,37 @@ class BeautifulSoupHTMLParser(HTMLParser):
                 value = ''
             attr_dict[key] = value
             attrvalue = '""'
-        self.soup.handle_starttag(name, None, None, attr_dict)
+        #print "START", name
+        sourceline, sourcepos = self.getpos()
+        tag = self.soup.handle_starttag(
+            name, None, None, attr_dict, sourceline=sourceline,
+            sourcepos=sourcepos
+        )
+        if tag and tag.is_empty_element and handle_empty_element:
+            # Unlike other parsers, html.parser doesn't send separate end tag
+            # events for empty-element tags. (It's handled in
+            # handle_startendtag, but only if the original markup looked like
+            # <tag/>.)
+            #
+            # So we need to call handle_endtag() ourselves. Since we
+            # know the start event is identical to the end event, we
+            # don't want handle_endtag() to cross off any previous end
+            # events for tags of this name.
+            self.handle_endtag(name, check_already_closed=False)
 
-    def handle_endtag(self, name):
+            # But we might encounter an explicit closing tag for this tag
+            # later on. If so, we want to ignore it.
+            self.already_closed_empty_element.append(name)
+            
+    def handle_endtag(self, name, check_already_closed=True):
+        #print "END", name
+        if check_already_closed and name in self.already_closed_empty_element:
+            # This is a redundant end tag for an empty-element tag.
+            # We've already called handle_endtag() for it, so just
+            # check it off the list.
+            # print "ALREADY CLOSED", name
+            self.already_closed_empty_element.remove(name)
+        else:
             self.soup.handle_endtag(name)
 
     def handle_data(self, data):
@@ -63,7 +136,8 @@ class BeautifulSoupHTMLParser(HTMLParser):
 
     def handle_charref(self, name):
         # XXX workaround for a bug in HTMLParser. Remove this once
-        # it's fixed.
+        # it's fixed in all supported versions.
+        # http://bugs.python.org/issue13633
         if name.startswith('x'):
             real_name = int(name.lstrip('x'), 16)
         elif name.startswith('X'):
@@ -71,11 +145,26 @@ class BeautifulSoupHTMLParser(HTMLParser):
         else:
             real_name = int(name)
 
+        data = None
+        if real_name < 256:
+            # HTML numeric entities are supposed to reference Unicode
+            # code points, but sometimes they reference code points in
+            # some other encoding (ahem, Windows-1252). E.g. &#147;
+            # instead of &#201; for LEFT DOUBLE QUOTATION MARK. This
+            # code tries to detect this situation and compensate.
+            for encoding in (self.soup.original_encoding, 'windows-1252'):
+                if not encoding:
+                    continue
                 try:
-            data = unichr(real_name)
+                    data = bytearray([real_name]).decode(encoding)
-        except (ValueError, OverflowError), e:
+                except UnicodeDecodeError as e:
-            data = u"\N{REPLACEMENT CHARACTER}"
+                    pass
-
+        if not data:
+            try:
+                data = chr(real_name)
+            except (ValueError, OverflowError) as e:
+                pass
+        data = data or "\N{REPLACEMENT CHARACTER}"
         self.handle_data(data)
 
     def handle_entityref(self, name):
@@ -83,7 +172,12 @@ class BeautifulSoupHTMLParser(HTMLParser):
         if character is not None:
             data = character
         else:
-            data = "&%s;" % name
+            # If this were XML, it would be ambiguous whether "&foo"
+            # was an character entity reference with a missing
+            # semicolon or the literal string "&foo". Since this is
+            # HTML, we have a complete list of all character entity references,
+            # and this one wasn't found, so assume it's the literal string "&foo".
+            data = "&%s" % name
         self.handle_data(data)
 
     def handle_comment(self, data):
@@ -113,14 +207,6 @@ class BeautifulSoupHTMLParser(HTMLParser):
 
     def handle_pi(self, data):
         self.soup.endData()
-        if data.endswith("?") and data.lower().startswith("xml"):
-            # "An XHTML processing instruction using the trailing '?'
-            # will cause the '?' to be included in data." - HTMLParser
-            # docs.
-            #
-            # Strip the question mark so we don't end up with two
-            # question marks.
-            data = data[:-1]
         self.soup.handle_data(data)
         self.soup.endData(ProcessingInstruction)
 
@@ -128,26 +214,38 @@ class BeautifulSoupHTMLParser(HTMLParser):
 class HTMLParserTreeBuilder(HTMLTreeBuilder):
 
     is_xml = False
-    features = [HTML, STRICT, HTMLPARSER]
+    picklable = True
+    NAME = HTMLPARSER
+    features = [NAME, HTML, STRICT]
 
-    def __init__(self, *args, **kwargs):
+    # The html.parser knows which line number and position in the
-        if CONSTRUCTOR_TAKES_STRICT:
+    # original file is the source of an element.
-            kwargs['strict'] = False
+    TRACKS_LINE_NUMBERS = True
-        self.parser_args = (args, kwargs)
+    
+    def __init__(self, parser_args=None, parser_kwargs=None, **kwargs):
+        super(HTMLParserTreeBuilder, self).__init__(**kwargs)
+        parser_args = parser_args or []
+        parser_kwargs = parser_kwargs or {}
+        if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED:
+            parser_kwargs['strict'] = False
+        if CONSTRUCTOR_TAKES_CONVERT_CHARREFS:
+            parser_kwargs['convert_charrefs'] = False
+        self.parser_args = (parser_args, parser_kwargs)
 
     def prepare_markup(self, markup, user_specified_encoding=None,
-                       document_declared_encoding=None):
+                       document_declared_encoding=None, exclude_encodings=None):
         """
         :return: A 4-tuple (markup, original encoding, encoding
         declared within markup, whether any characters had to be
         replaced with REPLACEMENT CHARACTER).
         """
-        if isinstance(markup, unicode):
+        if isinstance(markup, str):
             yield (markup, None, None, False)
             return
 
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
+        dammit = UnicodeDammit(markup, try_encodings, is_html=True,
+                               exclude_encodings=exclude_encodings)
         yield (dammit.markup, dammit.original_encoding,
                dammit.declared_html_encoding,
                dammit.contains_replacement_characters)
@@ -158,10 +256,12 @@ class HTMLParserTreeBuilder(HTMLTreeBuilder):
         parser.soup = self.soup
         try:
             parser.feed(markup)
-        except HTMLParseError, e:
+            parser.close()
+        except HTMLParseError as e:
             warnings.warn(RuntimeWarning(
                 "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
             raise e
+        parser.already_closed_empty_element = []
 
 # Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
 # 3.2.3 code. This ensures they don't treat markup like <p></p> as a

lib/bs4/builder/_lxml.py

@@ -1,13 +1,26 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'LXMLTreeBuilderForXML',
     'LXMLTreeBuilder',
     ]
 
+try:
+    from collections.abc import Callable # Python 3.6
+except ImportError as e:
+    from collections import Callable
+
 from io import BytesIO
-from StringIO import StringIO
+from io import StringIO
-import collections
 from lxml import etree
-from bs4.element import Comment, Doctype, NamespacedAttribute
+from bs4.element import (
+    Comment,
+    Doctype,
+    NamespacedAttribute,
+    ProcessingInstruction,
+    XMLProcessingInstruction,
+)
 from bs4.builder import (
     FAST,
     HTML,
@@ -20,19 +33,55 @@ from bs4.dammit import EncodingDetector
 
 LXML = 'lxml'
 
+def _invert(d):
+    "Invert a dictionary."
+    return dict((v,k) for k, v in list(d.items()))
+
 class LXMLTreeBuilderForXML(TreeBuilder):
     DEFAULT_PARSER_CLASS = etree.XMLParser
 
     is_xml = True
+    processing_instruction_class = XMLProcessingInstruction
+
+    NAME = "lxml-xml"
+    ALTERNATE_NAMES = ["xml"]
 
     # Well, it's permissive by XML parser standards.
-    features = [LXML, XML, FAST, PERMISSIVE]
+    features = [NAME, LXML, XML, FAST, PERMISSIVE]
 
     CHUNK_SIZE = 512
 
     # This namespace mapping is specified in the XML Namespace
     # standard.
-    DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"}
+    DEFAULT_NSMAPS = dict(xml='http://www.w3.org/XML/1998/namespace')
+
+    DEFAULT_NSMAPS_INVERTED = _invert(DEFAULT_NSMAPS)
+
+    # NOTE: If we parsed Element objects and looked at .sourceline,
+    # we'd be able to see the line numbers from the original document.
+    # But instead we build an XMLParser or HTMLParser object to serve
+    # as the target of parse messages, and those messages don't include
+    # line numbers.
+    
+    def initialize_soup(self, soup):
+        """Let the BeautifulSoup object know about the standard namespace
+        mapping.
+        """
+        super(LXMLTreeBuilderForXML, self).initialize_soup(soup)
+        self._register_namespaces(self.DEFAULT_NSMAPS)
+
+    def _register_namespaces(self, mapping):
+        """Let the BeautifulSoup object know about namespaces encountered
+        while parsing the document.
+
+        This might be useful later on when creating CSS selectors.
+        """
+        for key, value in list(mapping.items()):
+            if key and key not in self.soup._namespaces:
+                # Let the BeautifulSoup object know about a new namespace.
+                # If there are multiple namespaces defined with the same
+                # prefix, the first one in the document takes precedence.
+                self.soup._namespaces[key] = value
 
     def default_parser(self, encoding):
         # This can either return a parser object or a class, which
@@ -46,12 +95,12 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         # Use the default parser.
         parser = self.default_parser(encoding)
 
-        if isinstance(parser, collections.Callable):
+        if isinstance(parser, Callable):
             # Instantiate the parser with default arguments
             parser = parser(target=self, strip_cdata=False, encoding=encoding)
         return parser
 
-    def __init__(self, parser=None, empty_element_tags=None):
+    def __init__(self, parser=None, empty_element_tags=None, **kwargs):
         # TODO: Issue a warning if parser is present but not a
         # callable, since that means there's no way to create new
         # parsers for different encodings.
@@ -59,7 +108,8 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         if empty_element_tags is not None:
             self.empty_element_tags = set(empty_element_tags)
         self.soup = None
-        self.nsmaps = [self.DEFAULT_NSMAPS]
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
+        super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
         
     def _getNsTag(self, tag):
         # Split the namespace URL out of a fully-qualified lxml tag
@@ -70,6 +120,7 @@ class LXMLTreeBuilderForXML(TreeBuilder):
             return (None, tag)
 
     def prepare_markup(self, markup, user_specified_encoding=None,
+                       exclude_encodings=None,
                        document_declared_encoding=None):
         """
         :yield: A series of 4-tuples.
@@ -78,31 +129,37 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
         Each 4-tuple represents a strategy for parsing the document.
         """
-        if isinstance(markup, unicode):
-            # We were given Unicode. Maybe lxml can parse Unicode on
-            # this system?
-            yield markup, None, document_declared_encoding, False
-
-        if isinstance(markup, unicode):
-            # No, apparently not. Convert the Unicode to UTF-8 and
-            # tell lxml to parse it as UTF-8.
-            yield (markup.encode("utf8"), "utf8",
-                   document_declared_encoding, False)
-
         # Instead of using UnicodeDammit to convert the bytestring to
         # Unicode using different encodings, use EncodingDetector to
         # iterate over the encodings, and tell lxml to try to parse
         # the document as each one in turn.
         is_html = not self.is_xml
+        if is_html:
+            self.processing_instruction_class = ProcessingInstruction
+        else:
+            self.processing_instruction_class = XMLProcessingInstruction
+
+        if isinstance(markup, str):
+            # We were given Unicode. Maybe lxml can parse Unicode on
+            # this system?
+            yield markup, None, document_declared_encoding, False
+
+        if isinstance(markup, str):
+            # No, apparently not. Convert the Unicode to UTF-8 and
+            # tell lxml to parse it as UTF-8.
+            yield (markup.encode("utf8"), "utf8",
+                   document_declared_encoding, False)
+
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        detector = EncodingDetector(markup, try_encodings, is_html)
+        detector = EncodingDetector(
+            markup, try_encodings, is_html, exclude_encodings)
         for encoding in detector.encodings:
             yield (detector.markup, encoding, document_declared_encoding, False)
 
     def feed(self, markup):
         if isinstance(markup, bytes):
             markup = BytesIO(markup)
-        elif isinstance(markup, unicode):
+        elif isinstance(markup, str):
             markup = StringIO(markup)
 
         # Call feed() at least once, even if the markup is empty,
@@ -117,30 +174,36 @@ class LXMLTreeBuilderForXML(TreeBuilder):
                 if len(data) != 0:
                     self.parser.feed(data)
             self.parser.close()
-        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
+        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
-            raise ParserRejectedMarkup(str(e))
+            raise ParserRejectedMarkup(e)
 
     def close(self):
-        self.nsmaps = [self.DEFAULT_NSMAPS]
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
 
     def start(self, name, attrs, nsmap={}):
         # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
         attrs = dict(attrs)
         nsprefix = None
         # Invert each namespace map as it comes in.
-        if len(self.nsmaps) > 1:
+        if len(nsmap) == 0 and len(self.nsmaps) > 1:
                 # There are no new namespaces for this tag, but
                 # non-default namespaces are in play, so we need a
                 # separate tag stack to know when they end.
                 self.nsmaps.append(None)
         elif len(nsmap) > 0:
             # A new namespace mapping has come into play.
-            inverted_nsmap = dict((value, key) for key, value in nsmap.items())
+
-            self.nsmaps.append(inverted_nsmap)
+            # First, Let the BeautifulSoup object know about it.
+            self._register_namespaces(nsmap)
+
+            # Then, add it to our running list of inverted namespace
+            # mappings.
+            self.nsmaps.append(_invert(nsmap))
+
             # Also treat the namespace mapping as a set of attributes on the
             # tag, so we can recreate it later.
             attrs = attrs.copy()
-            for prefix, namespace in nsmap.items():
+            for prefix, namespace in list(nsmap.items()):
                 attribute = NamespacedAttribute(
                     "xmlns", prefix, "http://www.w3.org/2000/xmlns/")
                 attrs[attribute] = namespace
@@ -149,7 +212,7 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         # from lxml with namespaces attached to their names, and
         # turn then into NamespacedAttribute objects.
         new_attrs = {}
-        for attr, value in attrs.items():
+        for attr, value in list(attrs.items()):
             namespace, attr = self._getNsTag(attr)
             if namespace is None:
                 new_attrs[attr] = value
@@ -189,7 +252,9 @@ class LXMLTreeBuilderForXML(TreeBuilder):
             self.nsmaps.pop()
 
     def pi(self, target, data):
-        pass
+        self.soup.endData()
+        self.soup.handle_data(target + ' ' + data)
+        self.soup.endData(self.processing_instruction_class)
 
     def data(self, content):
         self.soup.handle_data(content)
@@ -207,13 +272,17 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""
-        return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
+        return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
 
 
 class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
 
-    features = [LXML, HTML, FAST, PERMISSIVE]
+    NAME = LXML
+    ALTERNATE_NAMES = ["lxml-html"]
+
+    features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE]
     is_xml = False
+    processing_instruction_class = ProcessingInstruction
 
     def default_parser(self, encoding):
         return etree.HTMLParser
@@ -224,10 +293,10 @@ class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
             self.parser = self.parser_for(encoding)
             self.parser.feed(markup)
             self.parser.close()
-        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
+        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
-            raise ParserRejectedMarkup(str(e))
+            raise ParserRejectedMarkup(e)
 
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""
-        return u'<html><body>%s</body></html>' % fragment
+        return '<html><body>%s</body></html>' % fragment

lib/bs4/check_block.py

@@ -0,0 +1,4 @@
+import requests
+data = requests.get("https://www.crummy.com/").content
+from bs4 import _s
+data = [x for x in _s(data).block_text()]

lib/bs4/dammit.py

@@ -3,12 +3,15 @@
 
 This library converts a bytestream to Unicode through any means
 necessary. It is heavily based on code from Mark Pilgrim's Universal
-Feed Parser. It works best on XML and XML, but it does not rewrite the
+Feed Parser. It works best on XML and HTML, but it does not rewrite the
 XML or HTML to reflect a new encoding; that's the tree builder's job.
 """
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
 
 import codecs
-from htmlentitydefs import codepoint2name
+from future.moves.html.entities import codepoint2name
+from future.builtins import chr
 import re
 import logging
 import string
@@ -20,6 +23,8 @@ try:
     #  PyPI package: cchardet
     import cchardet
     def chardet_dammit(s):
+        if isinstance(s, str):
+            return None
         return cchardet.detect(s)['encoding']
 except ImportError:
     try:
@@ -28,6 +33,8 @@ except ImportError:
         #  PyPI package: chardet
         import chardet
         def chardet_dammit(s):
+            if isinstance(s, str):
+                return None
             return chardet.detect(s)['encoding']
         #import chardet.constants
         #chardet.constants._debug = 1
@@ -42,10 +49,19 @@ try:
 except ImportError:
     pass
 
-xml_encoding_re = re.compile(
+# Build bytestring and Unicode versions of regular expressions for finding
-    '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
+# a declared encoding inside an XML or HTML document.
-html_meta_re = re.compile(
+xml_encoding = '^\s*<\\?.*encoding=[\'"](.*?)[\'"].*\\?>'
-    '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
+html_meta = '<\\s*meta[^>]+charset\\s*=\\s*["\']?([^>]*?)[ /;\'">]'
+encoding_res = dict()
+encoding_res[bytes] = {
+    'html' : re.compile(html_meta.encode("ascii"), re.I),
+    'xml' : re.compile(xml_encoding.encode("ascii"), re.I),
+}
+encoding_res[str] = {
+    'html' : re.compile(html_meta, re.I),
+    'xml' : re.compile(xml_encoding, re.I)
+}
 
 class EntitySubstitution(object):
 
@@ -55,15 +71,24 @@ class EntitySubstitution(object):
         lookup = {}
         reverse_lookup = {}
         characters_for_re = []
-        for codepoint, name in list(codepoint2name.items()):
+
-            character = unichr(codepoint)
+        # &apos is an XHTML entity and an HTML 5, but not an HTML 4
-            if codepoint != 34:
+        # entity. We don't want to use it, but we want to recognize it on the way in.
+        #
+        # TODO: Ideally we would be able to recognize all HTML 5 named
+        # entities, but that's a little tricky.
+        extra = [(39, 'apos')]
+        for codepoint, name in list(codepoint2name.items()) + extra:
+            character = chr(codepoint)
+            if codepoint not in (34, 39):
                 # There's no point in turning the quotation mark into
-                # &quot;, unless it happens within an attribute value, which
+                # &quot; or the single quote into &apos;, unless it
-                # is handled elsewhere.
+                # happens within an attribute value, which is handled
+                # elsewhere.
                 characters_for_re.append(character)
                 lookup[character] = name
-            # But we do want to turn &quot; into the quotation mark.
+            # But we do want to recognize those entities on the way in and
+            # convert them to Unicode characters.
             reverse_lookup[name] = character
         re_definition = "[%s]" % "".join(characters_for_re)
         return lookup, reverse_lookup, re.compile(re_definition)
@@ -79,7 +104,7 @@ class EntitySubstitution(object):
         }
 
     BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
-                                           "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
+                                           "&(?!#\\d+;|#x[0-9a-fA-F]+;|\\w+;)"
                                            ")")
 
     AMPERSAND_OR_BRACKET = re.compile("([<>&])")
@@ -212,8 +237,11 @@ class EncodingDetector:
 
     5. Windows-1252.
     """
-    def __init__(self, markup, override_encodings=None, is_html=False):
+    def __init__(self, markup, override_encodings=None, is_html=False,
+                 exclude_encodings=None):
         self.override_encodings = override_encodings or []
+        exclude_encodings = exclude_encodings or []
+        self.exclude_encodings = set([x.lower() for x in exclude_encodings])
         self.chardet_encoding = None
         self.is_html = is_html
         self.declared_encoding = None
@@ -224,6 +252,8 @@ class EncodingDetector:
     def _usable(self, encoding, tried):
         if encoding is not None:
             encoding = encoding.lower()
+            if encoding in self.exclude_encodings:
+                return False
             if encoding not in tried:
                 tried.add(encoding)
                 return True
@@ -266,6 +296,9 @@ class EncodingDetector:
     def strip_byte_order_mark(cls, data):
         """If a byte-order mark is present, strip it and return the encoding it implies."""
         encoding = None
+        if isinstance(data, str):
+            # Unicode data cannot have a byte-order mark.
+            return data, encoding
         if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
                and (data[2:4] != '\x00\x00'):
             encoding = 'utf-16be'
@@ -300,14 +333,22 @@ class EncodingDetector:
             xml_endpos = 1024
             html_endpos = max(2048, int(len(markup) * 0.05))
 
+        if isinstance(markup, bytes):
+            res = encoding_res[bytes]
+        else:
+            res = encoding_res[str]
+
+        xml_re = res['xml']
+        html_re = res['html']
         declared_encoding = None
-        declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos)
+        declared_encoding_match = xml_re.search(markup, endpos=xml_endpos)
         if not declared_encoding_match and is_html:
-            declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos)
+            declared_encoding_match = html_re.search(markup, endpos=html_endpos)
         if declared_encoding_match is not None:
-            declared_encoding = declared_encoding_match.groups()[0].decode(
+            declared_encoding = declared_encoding_match.groups()[0]
-                'ascii')
         if declared_encoding:
+            if isinstance(declared_encoding, bytes):
+                declared_encoding = declared_encoding.decode('ascii', 'replace')
             return declared_encoding.lower()
         return None
 
@@ -331,18 +372,19 @@ class UnicodeDammit:
         ]
 
     def __init__(self, markup, override_encodings=[],
-                 smart_quotes_to=None, is_html=False):
+                 smart_quotes_to=None, is_html=False, exclude_encodings=[]):
         self.smart_quotes_to = smart_quotes_to
         self.tried_encodings = []
         self.contains_replacement_characters = False
         self.is_html = is_html
-
+        self.log = logging.getLogger(__name__)
-        self.detector = EncodingDetector(markup, override_encodings, is_html)
+        self.detector = EncodingDetector(
+            markup, override_encodings, is_html, exclude_encodings)
 
         # Short-circuit if the data is in Unicode to begin with.
-        if isinstance(markup, unicode) or markup == '':
+        if isinstance(markup, str) or markup == '':
             self.markup = markup
-            self.unicode_markup = unicode(markup)
+            self.unicode_markup = str(markup)
             self.original_encoding = None
             return
 
@@ -365,9 +407,10 @@ class UnicodeDammit:
                 if encoding != "ascii":
                     u = self._convert_from(encoding, "replace")
                 if u is not None:
-                    logging.warning(
+                    self.log.warning(
                             "Some characters could not be decoded, and were "
-                            "replaced with REPLACEMENT CHARACTER.")
+                            "replaced with REPLACEMENT CHARACTER."
+                    )
                     self.contains_replacement_characters = True
                     break
 
@@ -425,7 +468,7 @@ class UnicodeDammit:
     def _to_unicode(self, data, encoding, errors="strict"):
         '''Given a string and its encoding, decodes the string into Unicode.
         %encoding is a string recognized by encodings.aliases'''
-        return unicode(data, encoding, errors)
+        return str(data, encoding, errors)
 
     @property
     def declared_html_encoding(self):

lib/bs4/diagnose.py

@@ -1,7 +1,11 @@
 """Diagnostic functions, mainly for use when doing tech support."""
+
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 import cProfile
-from StringIO import StringIO
+from io import StringIO
-from HTMLParser import HTMLParser
+from html.parser import HTMLParser
 import bs4
 from bs4 import BeautifulSoup, __version__
 from bs4.builder import builder_registry
@@ -17,8 +21,8 @@ import cProfile
 
 def diagnose(data):
     """Diagnostic suite for isolating common problems."""
-    print "Diagnostic running on Beautiful Soup %s" % __version__
+    print("Diagnostic running on Beautiful Soup %s" % __version__)
-    print "Python version %s" % sys.version
+    print("Python version %s" % sys.version)
 
     basic_parsers = ["html.parser", "html5lib", "lxml"]
     for name in basic_parsers:
@@ -27,44 +31,60 @@ def diagnose(data):
                 break
         else:
             basic_parsers.remove(name)
-            print (
+            print((
                 "I noticed that %s is not installed. Installing it may help." %
-                name)
+                name))
 
     if 'lxml' in basic_parsers:
-        basic_parsers.append(["lxml", "xml"])
+        basic_parsers.append("lxml-xml")
+        try:
             from lxml import etree
-        print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))
+            print("Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)))
+        except ImportError as e:
+            print (
+                "lxml is not installed or couldn't be imported.")
+
 
     if 'html5lib' in basic_parsers:
+        try:
             import html5lib
-        print "Found html5lib version %s" % html5lib.__version__
+            print("Found html5lib version %s" % html5lib.__version__)
+        except ImportError as e:
+            print (
+                "html5lib is not installed or couldn't be imported.")
 
     if hasattr(data, 'read'):
         data = data.read()
-    elif os.path.exists(data):
-        print '"%s" looks like a filename. Reading data from the file.' % data
-        data = open(data).read()
     elif data.startswith("http:") or data.startswith("https:"):
-        print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data
+        print('"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data)
-        print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup."
+        print("You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup.")
         return
-    print
+    else:
+        try:
+            if os.path.exists(data):
+                print('"%s" looks like a filename. Reading data from the file.' % data)
+                with open(data) as fp:
+                    data = fp.read()
+        except ValueError:
+            # This can happen on some platforms when the 'filename' is
+            # too long. Assume it's data and not a filename.
+            pass
+        print()
 
     for parser in basic_parsers:
-        print "Trying to parse your markup with %s" % parser
+        print("Trying to parse your markup with %s" % parser)
         success = False
         try:
-            soup = BeautifulSoup(data, parser)
+            soup = BeautifulSoup(data, features=parser)
             success = True
-        except Exception, e:
+        except Exception as e:
-            print "%s could not parse the markup." % parser
+            print("%s could not parse the markup." % parser)
             traceback.print_exc()
         if success:
-            print "Here's what %s did with the markup:" % parser
+            print("Here's what %s did with the markup:" % parser)
-            print soup.prettify()
+            print(soup.prettify())
 
-        print "-" * 80
+        print("-" * 80)
 
 def lxml_trace(data, html=True, **kwargs):
     """Print out the lxml events that occur during parsing.
@@ -74,7 +94,7 @@ def lxml_trace(data, html=True, **kwargs):
     """
     from lxml import etree
     for event, element in etree.iterparse(StringIO(data), html=html, **kwargs):
-        print("%s, %4s, %s" % (event, element.tag, element.text))
+        print(("%s, %4s, %s" % (event, element.tag, element.text)))
 
 class AnnouncingParser(HTMLParser):
     """Announces HTMLParser parse events, without doing anything else."""
@@ -156,9 +176,9 @@ def rdoc(num_elements=1000):
 
 def benchmark_parsers(num_elements=100000):
     """Very basic head-to-head performance benchmark."""
-    print "Comparative parser benchmark on Beautiful Soup %s" % __version__
+    print("Comparative parser benchmark on Beautiful Soup %s" % __version__)
     data = rdoc(num_elements)
-    print "Generated a large invalid HTML document (%d bytes)." % len(data)
+    print("Generated a large invalid HTML document (%d bytes)." % len(data))
     
     for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
         success = False
@@ -167,24 +187,24 @@ def benchmark_parsers(num_elements=100000):
             soup = BeautifulSoup(data, parser)
             b = time.time()
             success = True
-        except Exception, e:
+        except Exception as e:
-            print "%s could not parse the markup." % parser
+            print("%s could not parse the markup." % parser)
             traceback.print_exc()
         if success:
-            print "BS4+%s parsed the markup in %.2fs." % (parser, b-a)
+            print("BS4+%s parsed the markup in %.2fs." % (parser, b-a))
 
     from lxml import etree
     a = time.time()
     etree.HTML(data)
     b = time.time()
-    print "Raw lxml parsed the markup in %.2fs." % (b-a)
+    print("Raw lxml parsed the markup in %.2fs." % (b-a))
 
     import html5lib
     parser = html5lib.HTMLParser()
     a = time.time()
     parser.parse(data)
     b = time.time()
-    print "Raw html5lib parsed the markup in %.2fs." % (b-a)
+    print("Raw html5lib parsed the markup in %.2fs." % (b-a))
 
 def profile(num_elements=100000, parser="lxml"):
 

lib/bs4/formatter.py

@@ -0,0 +1,99 @@
+from bs4.dammit import EntitySubstitution
+
+class Formatter(EntitySubstitution):
+    """Describes a strategy to use when outputting a parse tree to a string.
+
+    Some parts of this strategy come from the distinction between
+    HTML4, HTML5, and XML. Others are configurable by the user.
+    """
+    # Registries of XML and HTML formatters.
+    XML_FORMATTERS = {}
+    HTML_FORMATTERS = {}
+
+    HTML = 'html'
+    XML = 'xml'
+
+    HTML_DEFAULTS = dict(
+        cdata_containing_tags=set(["script", "style"]),
+    )
+
+    def _default(self, language, value, kwarg):
+        if value is not None:
+            return value
+        if language == self.XML:
+            return set()
+        return self.HTML_DEFAULTS[kwarg]
+
+    def __init__(
+            self, language=None, entity_substitution=None,
+            void_element_close_prefix='/', cdata_containing_tags=None,
+    ):
+        """
+
+        :param void_element_close_prefix: By default, represent void
+        elements as <tag/> rather than <tag>
+        """
+        self.language = language
+        self.entity_substitution = entity_substitution
+        self.void_element_close_prefix = void_element_close_prefix
+        self.cdata_containing_tags = self._default(
+            language, cdata_containing_tags, 'cdata_containing_tags'
+        )
+            
+    def substitute(self, ns):
+        """Process a string that needs to undergo entity substitution."""
+        if not self.entity_substitution:
+            return ns
+        from .element import NavigableString
+        if (isinstance(ns, NavigableString)
+            and ns.parent is not None
+            and ns.parent.name in self.cdata_containing_tags):
+            # Do nothing.
+            return ns
+        # Substitute.
+        return self.entity_substitution(ns)
+
+    def attribute_value(self, value):
+        """Process the value of an attribute."""
+        return self.substitute(value)
+    
+    def attributes(self, tag):
+        """Reorder a tag's attributes however you want."""
+        return sorted(tag.attrs.items())
+
+   
+class HTMLFormatter(Formatter):
+    REGISTRY = {}
+    def __init__(self, *args, **kwargs):
+        return super(HTMLFormatter, self).__init__(self.HTML, *args, **kwargs)
+
+    
+class XMLFormatter(Formatter):
+    REGISTRY = {}
+    def __init__(self, *args, **kwargs):
+        return super(XMLFormatter, self).__init__(self.XML, *args, **kwargs)
+
+
+# Set up aliases for the default formatters.
+HTMLFormatter.REGISTRY['html'] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html
+)
+HTMLFormatter.REGISTRY["html5"] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html,
+    void_element_close_prefix = None
+)
+HTMLFormatter.REGISTRY["minimal"] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_xml
+)
+HTMLFormatter.REGISTRY[None] = HTMLFormatter(
+    entity_substitution=None
+)
+XMLFormatter.REGISTRY["html"] =  XMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html
+)
+XMLFormatter.REGISTRY["minimal"] = XMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_xml
+)
+XMLFormatter.REGISTRY[None] = Formatter(
+    Formatter(Formatter.XML, entity_substitution=None)
+)

lib/bs4/testing.py

@@ -1,5 +1,10 @@
+# encoding: utf-8
 """Helper classes for tests."""
 
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
+import pickle
 import copy
 import functools
 import unittest
@@ -11,29 +16,66 @@ from bs4.element import (
     ContentMetaAttributeValue,
     Doctype,
     SoupStrainer,
+    Tag
 )
 
 from bs4.builder import HTMLParserTreeBuilder
 default_builder = HTMLParserTreeBuilder
 
+BAD_DOCUMENT = """A bare string
+<!DOCTYPE xsl:stylesheet SYSTEM "htmlent.dtd">
+<!DOCTYPE xsl:stylesheet PUBLIC "htmlent.dtd">
+<div><![CDATA[A CDATA section where it doesn't belong]]></div>
+<div><svg><![CDATA[HTML5 does allow CDATA sections in SVG]]></svg></div>
+<div>A <meta> tag</div>
+<div>A <br> tag that supposedly has contents.</br></div>
+<div>AT&T</div>
+<div><textarea>Within a textarea, markup like <b> tags and <&<&amp; should be treated as literal</textarea></div>
+<div><script>if (i < 2) { alert("<b>Markup within script tags should be treated as literal.</b>"); }</script></div>
+<div>This numeric entity is missing the final semicolon: <x t="pi&#241ata"></div>
+<div><a href="http://example.com/</a> that attribute value never got closed</div>
+<div><a href="foo</a>, </a><a href="bar">that attribute value was closed by the subsequent tag</a></div>
+<! This document starts with a bogus declaration ><div>a</div>
+<div>This document contains <!an incomplete declaration <div>(do you see it?)</div>
+<div>This document ends with <!an incomplete declaration
+<div><a style={height:21px;}>That attribute value was bogus</a></div>
+<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">The doctype is invalid because it contains extra whitespace
+<div><table><td nowrap>That boolean attribute had no value</td></table></div>
+<div>Here's a nonexistent entity: &#foo; (do you see it?)</div>
+<div>This document ends before the entity finishes: &gt
+<div><p>Paragraphs shouldn't contain block display elements, but this one does: <dl><dt>you see?</dt></p>
+<b b="20" a="1" b="10" a="2" a="3" a="4">Multiple values for the same attribute.</b>
+<div><table><tr><td>Here's a table</td></tr></table></div>
+<div><table id="1"><tr><td>Here's a nested table:<table id="2"><tr><td>foo</td></tr></table></td></div>
+<div>This tag contains nothing but whitespace: <b>    </b></div>
+<div><blockquote><p><b>This p tag is cut off by</blockquote></p>the end of the blockquote tag</div>
+<div><table><div>This table contains bare markup</div></table></div>
+<div><div id="1">\n <a href="link1">This link is never closed.\n</div>\n<div id="2">\n <div id="3">\n   <a href="link2">This link is closed.</a>\n  </div>\n</div></div>
+<div>This document contains a <!DOCTYPE surprise>surprise doctype</div>
+<div><a><B><Cd><EFG>Mixed case tags are folded to lowercase</efg></CD></b></A></div>
+<div><our\u2603>Tag name contains Unicode characters</our\u2603></div>
+<div><a \u2603="snowman">Attribute name contains Unicode characters</a></div>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+"""
+
 
 class SoupTest(unittest.TestCase):
 
     @property
     def default_builder(self):
-        return default_builder()
+        return default_builder
 
     def soup(self, markup, **kwargs):
         """Build a Beautiful Soup object from markup."""
         builder = kwargs.pop('builder', self.default_builder)
         return BeautifulSoup(markup, builder=builder, **kwargs)
 
-    def document_for(self, markup):
+    def document_for(self, markup, **kwargs):
         """Turn an HTML fragment into a document.
 
         The details depend on the builder.
         """
-        return self.default_builder.test_fragment_to_document(markup)
+        return self.default_builder(**kwargs).test_fragment_to_document(markup)
 
     def assertSoupEquals(self, to_parse, compare_parsed_to=None):
         builder = self.default_builder
@@ -43,6 +85,131 @@ class SoupTest(unittest.TestCase):
 
         self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
 
+    def assertConnectedness(self, element):
+        """Ensure that next_element and previous_element are properly
+        set for all descendants of the given element.
+        """
+        earlier = None
+        for e in element.descendants:
+            if earlier:
+                self.assertEqual(e, earlier.next_element)
+                self.assertEqual(earlier, e.previous_element)
+            earlier = e
+
+    def linkage_validator(self, el, _recursive_call=False):
+        """Ensure proper linkage throughout the document."""
+        descendant = None
+        # Document element should have no previous element or previous sibling.
+        # It also shouldn't have a next sibling.
+        if el.parent is None:
+            assert el.previous_element is None,\
+                "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format(
+                    el, el.previous_element, None
+                )
+            assert el.previous_sibling is None,\
+                "Bad previous_sibling\nNODE: {}\nPREV: {}\nEXPECTED: {}".format(
+                    el, el.previous_sibling, None
+                )
+            assert el.next_sibling is None,\
+                "Bad next_sibling\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format(
+                    el, el.next_sibling, None
+                )
+
+        idx = 0
+        child = None
+        last_child = None
+        last_idx = len(el.contents) - 1
+        for child in el.contents:
+            descendant = None
+
+            # Parent should link next element to their first child
+            # That child should have no previous sibling
+            if idx == 0:
+                if el.parent is not None:
+                    assert el.next_element is child,\
+                       "Bad next_element\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format(
+                            el, el.next_element, child
+                        )
+                    assert child.previous_element is el,\
+                       "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format(
+                            child, child.previous_element, el
+                        )
+                    assert child.previous_sibling is None,\
+                       "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED: {}".format(
+                            child, child.previous_sibling, None
+                        )
+
+            # If not the first child, previous index should link as sibling to this index
+            # Previous element should match the last index or the last bubbled up descendant
+            else:
+                assert child.previous_sibling is el.contents[idx - 1],\
+                    "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED {}".format(
+                        child, child.previous_sibling, el.contents[idx - 1]
+                    )
+                assert el.contents[idx - 1].next_sibling is child,\
+                    "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                        el.contents[idx - 1], el.contents[idx - 1].next_sibling, child
+                    )
+
+                if last_child is not None:
+                    assert child.previous_element is last_child,\
+                        "Bad previous_element\nNODE: {}\nPREV {}\nEXPECTED {}\nCONTENTS {}".format(
+                            child, child.previous_element, last_child, child.parent.contents
+                        )
+                    assert last_child.next_element is child,\
+                        "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                            last_child, last_child.next_element, child
+                        )
+
+            if isinstance(child, Tag) and child.contents:
+                descendant = self.linkage_validator(child, True)
+                # A bubbled up descendant should have no next siblings
+                assert descendant.next_sibling is None,\
+                    "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                        descendant, descendant.next_sibling, None
+                    )
+
+            # Mark last child as either the bubbled up descendant or the current child
+            if descendant is not None:
+                last_child = descendant
+            else:
+                last_child = child
+
+            # If last child, there are non next siblings
+            if idx == last_idx:
+                assert child.next_sibling is None,\
+                    "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                        child, child.next_sibling, None
+                    )
+            idx += 1
+
+        child = descendant if descendant is not None else child
+        if child is None:
+            child = el
+
+        if not _recursive_call and child is not None:
+            target = el
+            while True:
+                if target is None:
+                    assert child.next_element is None, \
+                        "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                            child, child.next_element, None
+                        )
+                    break
+                elif target.next_sibling is not None:
+                    assert child.next_element is target.next_sibling, \
+                        "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format(
+                            child, child.next_element, target.next_sibling
+                        )
+                    break
+                target = target.parent
+
+            # We are done, so nothing to return
+            return None
+        else:
+            # Return the child to the recursive caller
+            return child
+
 
 class HTMLTreeBuilderSmokeTest(object):
 
@@ -54,6 +221,27 @@ class HTMLTreeBuilderSmokeTest(object):
     markup in these tests, there's not much room for interpretation.
     """
 
+    def test_empty_element_tags(self):
+        """Verify that all HTML4 and HTML5 empty element (aka void element) tags
+        are handled correctly.
+        """
+        for name in [
+                'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
+                'spacer', 'frame'
+        ]:
+            soup = self.soup("")
+            new_tag = soup.new_tag(name)
+            self.assertEqual(True, new_tag.is_empty_element)
+
+    def test_pickle_and_unpickle_identity(self):
+        # Pickling a tree, then unpickling it, yields a tree identical
+        # to the original.
+        tree = self.soup("<a><b>foo</a>")
+        dumped = pickle.dumps(tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertEqual(loaded.__class__, BeautifulSoup)
+        self.assertEqual(loaded.decode(), tree.decode())
+
     def assertDoctypeHandled(self, doctype_fragment):
         """Assert that a given doctype string is handled correctly."""
         doctype_str, soup = self._document_with_doctype(doctype_fragment)
@@ -114,6 +302,27 @@ class HTMLTreeBuilderSmokeTest(object):
             soup.encode("utf-8").replace(b"\n", b""),
             markup.replace(b"\n", b""))
 
+    def test_namespaced_html(self):
+        """When a namespaced XML document is parsed as HTML it should
+        be treated as HTML with weird tag names.
+        """
+        markup = b"""<ns1:foo>content</ns1:foo><ns1:foo/><ns2:foo/>"""
+        soup = self.soup(markup)
+        self.assertEqual(2, len(soup.find_all("ns1:foo")))
+        
+    def test_processing_instruction(self):
+        # We test both Unicode and bytestring to verify that
+        # process_markup correctly sets processing_instruction_class
+        # even when the markup is already Unicode and there is no
+        # need to process anything.
+        markup = """<?PITarget PIContent?>"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.decode())
+
+        markup = b"""<?PITarget PIContent?>"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode("utf8"))
+
     def test_deepcopy(self):
         """Make sure you can copy the tree builder.
 
@@ -155,6 +364,23 @@ class HTMLTreeBuilderSmokeTest(object):
     def test_nested_formatting_elements(self):
         self.assertSoupEquals("<em><em></em></em>")
 
+    def test_double_head(self):
+        html = '''<!DOCTYPE html>
+<html>
+<head>
+<title>Ordinary HEAD element test</title>
+</head>
+<script type="text/javascript">
+alert("Help!");
+</script>
+<body>
+Hello, world!
+</body>
+</html>
+'''
+        soup = self.soup(html)
+        self.assertEqual("text/javascript", soup.find('script')['type'])
+
     def test_comment(self):
         # Comments are represented as Comment objects.
         markup = "<p>foo<!--foobar-->baz</p>"
@@ -171,9 +397,22 @@ class HTMLTreeBuilderSmokeTest(object):
         self.assertEqual(comment, baz.previous_element)
 
     def test_preserved_whitespace_in_pre_and_textarea(self):
-        """Whitespace must be preserved in <pre> and <textarea> tags."""
+        """Whitespace must be preserved in <pre> and <textarea> tags,
-        self.assertSoupEquals("<pre>   </pre>")
+        even if that would mean not prettifying the markup.
-        self.assertSoupEquals("<textarea> woo  </textarea>")
+        """
+        pre_markup = "<pre>   </pre>"
+        textarea_markup = "<textarea> woo\nwoo  </textarea>"
+        self.assertSoupEquals(pre_markup)
+        self.assertSoupEquals(textarea_markup)
+
+        soup = self.soup(pre_markup)
+        self.assertEqual(soup.pre.prettify(), pre_markup)
+
+        soup = self.soup(textarea_markup)
+        self.assertEqual(soup.textarea.prettify(), textarea_markup)
+
+        soup = self.soup("<textarea></textarea>")
+        self.assertEqual(soup.textarea.prettify(), "<textarea></textarea>")
 
     def test_nested_inline_elements(self):
         """Inline elements can be nested indefinitely."""
@@ -213,6 +452,18 @@ class HTMLTreeBuilderSmokeTest(object):
             "<tbody><tr><td>Bar</td></tr></tbody>"
             "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
 
+    def test_multivalued_attribute_with_whitespace(self):
+        # Whitespace separating the values of a multi-valued attribute
+        # should be ignored.
+
+        markup = '<div class=" foo bar	 "></a>'
+        soup = self.soup(markup)
+        self.assertEqual(['foo', 'bar'], soup.div['class'])
+
+        # If you search by the literal name of the class it's like the whitespace
+        # wasn't there.
+        self.assertEqual(soup.div, soup.find('div', class_="foo bar"))
+        
     def test_deeply_nested_multivalued_attribute(self):
         # html5lib can set the attributes of the same tag many times
         # as it rearranges the tree. This has caused problems with
@@ -221,18 +472,52 @@ class HTMLTreeBuilderSmokeTest(object):
         soup = self.soup(markup)
         self.assertEqual(["css"], soup.div.div['class'])
 
+    def test_multivalued_attribute_on_html(self):
+        # html5lib uses a different API to set the attributes ot the
+        # <html> tag. This has caused problems with multivalued
+        # attributes.
+        markup = '<html class="a b"></html>'
+        soup = self.soup(markup)
+        self.assertEqual(["a", "b"], soup.html['class'])
+
     def test_angle_brackets_in_attribute_values_are_escaped(self):
         self.assertSoupEquals('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')
 
+    def test_strings_resembling_character_entity_references(self):
+        # "&T" and "&p" look like incomplete character entities, but they are
+        # not.
+        self.assertSoupEquals(
+            "<p>&bull; AT&T is in the s&p 500</p>",
+            "<p>\u2022 AT&amp;T is in the s&amp;p 500</p>"
+        )
+
+    def test_apos_entity(self):
+        self.assertSoupEquals(
+            "<p>Bob&apos;s Bar</p>",
+            "<p>Bob's Bar</p>",
+        )
+        
+    def test_entities_in_foreign_document_encoding(self):
+        # &#147; and &#148; are invalid numeric entities referencing
+        # Windows-1252 characters. &#45; references a character common
+        # to Windows-1252 and Unicode, and &#9731; references a
+        # character only found in Unicode.
+        #
+        # All of these entities should be converted to Unicode
+        # characters.
+        markup = "<p>&#147;Hello&#148; &#45;&#9731;</p>"
+        soup = self.soup(markup)
+        self.assertEqual("“Hello” -☃", soup.p.string)
+        
     def test_entities_in_attributes_converted_to_unicode(self):
-        expect = u'<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
+        expect = '<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
         self.assertSoupEquals('<p id="pi&#241;ata"></p>', expect)
         self.assertSoupEquals('<p id="pi&#xf1;ata"></p>', expect)
         self.assertSoupEquals('<p id="pi&#Xf1;ata"></p>', expect)
         self.assertSoupEquals('<p id="pi&ntilde;ata"></p>', expect)
 
     def test_entities_in_text_converted_to_unicode(self):
-        expect = u'<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
+        expect = '<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
         self.assertSoupEquals("<p>pi&#241;ata</p>", expect)
         self.assertSoupEquals("<p>pi&#xf1;ata</p>", expect)
         self.assertSoupEquals("<p>pi&#Xf1;ata</p>", expect)
@@ -243,7 +528,7 @@ class HTMLTreeBuilderSmokeTest(object):
                               '<p>I said "good day!"</p>')
 
     def test_out_of_range_entity(self):
-        expect = u"\N{REPLACEMENT CHARACTER}"
+        expect = "\N{REPLACEMENT CHARACTER}"
         self.assertSoupEquals("&#10000000000000;", expect)
         self.assertSoupEquals("&#x10000000000000;", expect)
         self.assertSoupEquals("&#1000000000;", expect)
@@ -253,6 +538,42 @@ class HTMLTreeBuilderSmokeTest(object):
         soup = self.soup("<html><h2>\nfoo</h2><p></p></html>")
         self.assertEqual("p", soup.h2.string.next_element.name)
         self.assertEqual("p", soup.p.name)
+        self.assertConnectedness(soup)
+
+    def test_empty_element_tags(self):
+        """Verify consistent handling of empty-element tags,
+        no matter how they come in through the markup.
+        """
+        self.assertSoupEquals('<br/><br/><br/>', "<br/><br/><br/>")
+        self.assertSoupEquals('<br /><br /><br />', "<br/><br/><br/>")
+        
+    def test_head_tag_between_head_and_body(self):
+        "Prevent recurrence of a bug in the html5lib treebuilder."
+        content = """<html><head></head>
+  <link></link>
+  <body>foo</body>
+</html>
+"""
+        soup = self.soup(content)
+        self.assertNotEqual(None, soup.html.body)
+        self.assertConnectedness(soup)
+
+    def test_multiple_copies_of_a_tag(self):
+        "Prevent recurrence of a bug in the html5lib treebuilder."
+        content = """<!DOCTYPE html>
+<html>
+ <body>
+   <article id="a" >
+   <div><a href="1"></div>
+   <footer>
+     <a href="2"></a>
+   </footer>
+  </article>
+  </body>
+</html>
+"""
+        soup = self.soup(content)
+        self.assertConnectedness(soup.article)
 
     def test_basic_namespaces(self):
         """Parsers don't need to *understand* namespaces, but at the
@@ -285,9 +606,9 @@ class HTMLTreeBuilderSmokeTest(object):
         # A seemingly innocuous document... but it's in Unicode! And
         # it contains characters that can't be represented in the
         # encoding found in the  declaration! The horror!
-        markup = u'<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>'
+        markup = '<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>'
         soup = self.soup(markup)
-        self.assertEqual(u'Sacr\xe9 bleu!', soup.body.string)
+        self.assertEqual('Sacr\xe9 bleu!', soup.body.string)
 
     def test_soupstrainer(self):
         """Parsers should be able to work with SoupStrainers."""
@@ -327,7 +648,7 @@ class HTMLTreeBuilderSmokeTest(object):
         # Both XML and HTML entities are converted to Unicode characters
         # during parsing.
         text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
-        expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
+        expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
         self.assertSoupEquals(text, expected)
 
     def test_smart_quotes_converted_on_the_way_in(self):
@@ -337,15 +658,15 @@ class HTMLTreeBuilderSmokeTest(object):
         soup = self.soup(quote)
         self.assertEqual(
             soup.p.string,
-            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
+            "\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
 
     def test_non_breaking_spaces_converted_on_the_way_in(self):
         soup = self.soup("<a>&nbsp;&nbsp;</a>")
-        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
+        self.assertEqual(soup.a.string, "\N{NO-BREAK SPACE}" * 2)
 
     def test_entities_converted_on_the_way_out(self):
         text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
-        expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
+        expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
         soup = self.soup(text)
         self.assertEqual(soup.p.encode("utf-8"), expected)
 
@@ -354,7 +675,7 @@ class HTMLTreeBuilderSmokeTest(object):
         # easy-to-understand document.
 
         # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
-        unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
+        unicode_html = '<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
 
         # That's because we're going to encode it into ISO-Latin-1, and use
         # that to test.
@@ -399,7 +720,9 @@ class HTMLTreeBuilderSmokeTest(object):
         hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
         soup = self.soup(
             hebrew_document, from_encoding="iso8859-8")
-        self.assertEqual(soup.original_encoding, 'iso8859-8')
+        # Some tree builders call it iso8859-8, others call it iso-8859-9.
+        # That's not a difference we really care about.
+        assert soup.original_encoding in ('iso8859-8', 'iso-8859-8')
         self.assertEqual(
             soup.encode('utf-8'),
             hebrew_document.decode("iso8859-8").encode("utf-8"))
@@ -461,13 +784,39 @@ class HTMLTreeBuilderSmokeTest(object):
         data.a['foo'] = 'bar'
         self.assertEqual('<a foo="bar">text</a>', data.a.decode())
 
+    def test_worst_case(self):
+        """Test the worst case (currently) for linking issues."""
+
+        soup = self.soup(BAD_DOCUMENT)
+        self.linkage_validator(soup)
+
+
 class XMLTreeBuilderSmokeTest(object):
 
+    def test_pickle_and_unpickle_identity(self):
+        # Pickling a tree, then unpickling it, yields a tree identical
+        # to the original.
+        tree = self.soup("<a><b>foo</a>")
+        dumped = pickle.dumps(tree, 2)
+        loaded = pickle.loads(dumped)
+        self.assertEqual(loaded.__class__, BeautifulSoup)
+        self.assertEqual(loaded.decode(), tree.decode())
+
     def test_docstring_generated(self):
         soup = self.soup("<root/>")
         self.assertEqual(
             soup.encode(), b'<?xml version="1.0" encoding="utf-8"?>\n<root/>')
 
+    def test_xml_declaration(self):
+        markup = b"""<?xml version="1.0" encoding="utf8"?>\n<foo/>"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode("utf8"))
+
+    def test_processing_instruction(self):
+        markup = b"""<?xml version="1.0" encoding="utf8"?>\n<?PITarget PIContent?>"""
+        soup = self.soup(markup)
+        self.assertEqual(markup, soup.encode("utf8"))
+
     def test_real_xhtml_document(self):
         """A real XHTML document should come out *exactly* the same as it went in."""
         markup = b"""<?xml version="1.0" encoding="utf-8"?>
@@ -480,12 +829,23 @@ class XMLTreeBuilderSmokeTest(object):
         self.assertEqual(
             soup.encode("utf-8"), markup)
 
+    def test_nested_namespaces(self):
+        doc = b"""<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<parent xmlns="http://ns1/">
+<child xmlns="http://ns2/" xmlns:ns3="http://ns3/">
+<grandchild ns3:attr="value" xmlns="http://ns4/"/>
+</child>
+</parent>"""
+        soup = self.soup(doc)
+        self.assertEqual(doc, soup.encode())
+        
     def test_formatter_processes_script_tag_for_xml_documents(self):
         doc = """
   <script type="text/javascript">
   </script>
 """
-        soup = BeautifulSoup(doc, "xml")
+        soup = BeautifulSoup(doc, "lxml-xml")
         # lxml would have stripped this while parsing, but we can add
         # it later.
         soup.script.string = 'console.log("< < hey > > ");'
@@ -493,15 +853,15 @@ class XMLTreeBuilderSmokeTest(object):
         self.assertTrue(b"&lt; &lt; hey &gt; &gt;" in encoded)
 
     def test_can_parse_unicode_document(self):
-        markup = u'<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
+        markup = '<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
         soup = self.soup(markup)
-        self.assertEqual(u'Sacr\xe9 bleu!', soup.root.string)
+        self.assertEqual('Sacr\xe9 bleu!', soup.root.string)
 
     def test_popping_namespaced_tag(self):
         markup = '<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>'
         soup = self.soup(markup)
         self.assertEqual(
-            unicode(soup.rss), markup)
+            str(soup.rss), markup)
 
     def test_docstring_includes_correct_encoding(self):
         soup = self.soup("<root/>")
@@ -532,17 +892,57 @@ class XMLTreeBuilderSmokeTest(object):
     def test_closing_namespaced_tag(self):
         markup = '<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>'
         soup = self.soup(markup)
-        self.assertEqual(unicode(soup.p), markup)
+        self.assertEqual(str(soup.p), markup)
 
     def test_namespaced_attributes(self):
         markup = '<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>'
         soup = self.soup(markup)
-        self.assertEqual(unicode(soup.foo), markup)
+        self.assertEqual(str(soup.foo), markup)
 
     def test_namespaced_attributes_xml_namespace(self):
         markup = '<foo xml:lang="fr">bar</foo>'
         soup = self.soup(markup)
-        self.assertEqual(unicode(soup.foo), markup)
+        self.assertEqual(str(soup.foo), markup)
+
+    def test_find_by_prefixed_name(self):
+        doc = """<?xml version="1.0" encoding="utf-8"?>
+<Document xmlns="http://example.com/ns0"
+    xmlns:ns1="http://example.com/ns1"
+    xmlns:ns2="http://example.com/ns2"
+    <ns1:tag>foo</ns1:tag>
+    <ns1:tag>bar</ns1:tag>
+    <ns2:tag key="value">baz</ns2:tag>
+</Document>
+"""
+        soup = self.soup(doc)
+
+        # There are three <tag> tags.
+        self.assertEqual(3, len(soup.find_all('tag')))
+
+        # But two of them are ns1:tag and one of them is ns2:tag.
+        self.assertEqual(2, len(soup.find_all('ns1:tag')))
+        self.assertEqual(1, len(soup.find_all('ns2:tag')))
+        
+        self.assertEqual(1, len(soup.find_all('ns2:tag', key='value')))
+        self.assertEqual(3, len(soup.find_all(['ns1:tag', 'ns2:tag'])))
+        
+    def test_copy_tag_preserves_namespace(self):
+        xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<w:document xmlns:w="http://example.com/ns0"/>"""
+    
+        soup = self.soup(xml)
+        tag = soup.document
+        duplicate = copy.copy(tag)
+
+        # The two tags have the same namespace prefix.
+        self.assertEqual(tag.prefix, duplicate.prefix)
+
+    def test_worst_case(self):
+        """Test the worst case (currently) for linking issues."""
+
+        soup = self.soup(BAD_DOCUMENT)
+        self.linkage_validator(soup)
+
 
 class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
     """Smoke test for a tree builder that supports HTML5."""

lib/bs4/tests/__init__.py

@@ -0,0 +1 @@
+"The beautifulsoup tests."