v2.5.2

This reverts commit cd8a899521.

.dockerignore

@@ -1,5 +1,10 @@
 .git
 .github
 .gitignore
+contrib
+init-scripts
+package
+pylintrc
 *.md
 !CHANGELOG*.md
+start.bat

.github/workflows/publish-docker.yml

@@ -0,0 +1,95 @@
+name: Publish Docker
+on:
+  push:
+    branches: [master, beta, nightly, python3]
+    tags: [v*]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Prepare
+        id: prepare
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo ::set-output name=tag::${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/master ]]; then
+            echo ::set-output name=tag::latest
+          else
+            echo ::set-output name=tag::${GITHUB_REF#refs/heads/}
+          fi
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo ::set-output name=branch::master
+          else
+            echo ::set-output name=branch::${GITHUB_REF#refs/heads/}
+          fi
+          echo ::set-output name=commit::${GITHUB_SHA}
+          echo ::set-output name=build_date::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+          echo ::set-output name=docker_platforms::linux/amd64,linux/arm64,linux/arm
+          echo ::set-output name=docker_image::${{ secrets.DOCKER_REPO }}/tautulli
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: crazy-max/ghaction-docker-buildx@v3
+        with:
+          buildx-version: latest
+
+      - name: Cache Docker Layers
+        id: cache
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Docker Buildx (no push)
+        run: |
+          docker buildx build \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --cache-to "type=local,dest=/tmp/.buildx-cache" \
+            --platform ${{ steps.prepare.outputs.docker_platforms }} \
+            --output "type=image,push=false" \
+            --build-arg "TAG=${{ steps.prepare.outputs.tag }}" \
+            --build-arg "BRANCH=${{ steps.prepare.outputs.branch }}" \
+            --build-arg "COMMIT=${{ steps.prepare.outputs.commit }}" \
+            --build-arg "BUILD_DATE=${{ steps.prepare.outputs.build_date }}" \
+            --tag "${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.tag }}" \
+            --file Dockerfile .
+
+      - name: Docker Login
+        if: success()
+        env:
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          echo "${DOCKER_PASSWORD}" | docker login --username "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+
+      - name: Docker Buildx (push)
+        if: success()
+        run: |
+          docker buildx build \
+            --cache-from "type=local,src=/tmp/.buildx-cache" \
+            --platform ${{ steps.prepare.outputs.docker_platforms }} \
+            --output "type=image,push=true" \
+            --build-arg "TAG=${{ steps.prepare.outputs.tag }}" \
+            --build-arg "BRANCH=${{ steps.prepare.outputs.branch }}" \
+            --build-arg "COMMIT=${{ steps.prepare.outputs.commit }}" \
+            --build-arg "BUILD_DATE=${{ steps.prepare.outputs.build_date }}" \
+            --tag "${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.tag }}" \
+            --file Dockerfile .
+
+      - name: Clear
+        if: always()
+        run: |
+          rm -f ${HOME}/.docker/config.json
+
+      - name: Post Status to Discord
+        uses: sarisia/actions-status-discord@v1
+        if: always()
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          status: ${{ job.status }}
+          title: ${{ github.workflow }}
+          nofail: true

.github/workflows/publish-release.yml

@@ -0,0 +1,204 @@
+name: Publish Release
+on:
+  push:
+    branches: [master, beta, nightly, python3]
+    tags: [v*]
+
+jobs:
+  build-windows:
+    runs-on: windows-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Set Release Version
+        id: get_version
+        shell: bash
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION_NSIS=${GITHUB_REF#refs/tags/v}.1
+            echo ::set-output name=VERSION_NSIS::${VERSION_NSIS/%-beta.1/.0}
+            echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/v}
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+          else
+            echo ::set-output name=VERSION_NSIS::0.0.0.0
+            echo ::set-output name=VERSION::0.0.0
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_SHA::7}
+          fi
+          echo $GITHUB_SHA > version.txt
+
+      - name: Set Up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Cache Dependencies
+        id: cache_dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{ runner.os }}-pip-${{ hashFiles('package/requirements-windows.txt') }}
+          restore-keys: ${{ runner.os }}-pip-
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r package/requirements-windows.txt
+
+      - name: Build Package
+        run: |
+          pyinstaller -y ./package/Tautulli-windows.spec
+
+      - name: Create Installer
+        uses: joncloud/makensis-action@v1.2
+        with:
+          script-file: ./package/Tautulli.nsi
+          arguments: /DVERSION=${{ steps.get_version.outputs.VERSION_NSIS }} /DINSTALLER_NAME=..\Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}.exe
+          include-more-plugins: true
+          include-custom-plugins-path: package/nsis-plugins
+
+      - name: Upload Installer
+        uses: actions/upload-artifact@v2
+        with:
+          name: Tautulli-windows-installer
+          path: Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}.exe
+
+      - name: Post Status to Discord
+        uses: sarisia/actions-status-discord@v1
+        if: always()
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          status: ${{ job.status }}
+          title: Build Windows Installer
+          nofail: true
+
+  build-macos:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Set Release Version
+        id: get_version
+        shell: bash
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo ::set-env name=VERSION::${GITHUB_REF#refs/tags/v}
+            echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/v}
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+          else
+            echo ::set-env name=VERSION::0.0.0
+            echo ::set-output name=VERSION::0.0.0
+            echo ::set-output name=RELEASE_VERSION::${GITHUB_SHA::7}
+          fi
+          echo $GITHUB_SHA > version.txt
+
+      - name: Set Up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Cache Dependencies
+        id: cache_dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/Library/Caches/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('package/requirements-macos.txt') }}
+          restore-keys: ${{ runner.os }}-pip-
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r package/requirements-macos.txt
+
+      - name: Build Package
+        run: |
+          pyinstaller -y ./package/Tautulli-macos.spec
+
+      - name: Create Installer
+        run: |
+          sudo pkgbuild --install-location /Applications --version ${{ steps.get_version.outputs.VERSION }} --component ./dist/Tautulli.app --scripts ./package/macos-scripts Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}.pkg
+
+      - name: Upload Installer
+        uses: actions/upload-artifact@v2
+        with:
+          name: Tautulli-macos-installer
+          path: Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}.pkg
+
+      - name: Post Status to Discord
+        uses: sarisia/actions-status-discord@v1
+        if: always()
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          status: ${{ job.status }}
+          title: Build MacOS Installer
+          nofail: true
+
+  release:
+    needs: [build-windows, build-macos]
+    if: startsWith(github.ref, 'refs/tags/') && always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get Build Job Status
+        uses: technote-space/workflow-conclusion-action@v1
+
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Set Release Version
+        id: get_version
+        run: |
+          echo ::set-output name=RELEASE_VERSION::${GITHUB_REF#refs/tags/}
+
+      - name: Download Windows Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/download-artifact@v2
+        with:
+          name: Tautulli-windows-installer
+
+      - name: Download MacOS Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/download-artifact@v2
+        with:
+          name: Tautulli-macos-installer
+
+      - name: Get Changelog
+        id: get_changelog
+        run: echo ::set-output name=CHANGELOG::"$( sed -n '/^## /{p; :loop n; p; /^## /q; b loop}' CHANGELOG.md | sed '$d' | sed '$d' | sed '$d' | sed ':a;N;$!ba;s/\n/%0A/g' )"
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.get_version.outputs.RELEASE_VERSION }}
+          release_name: Tautulli ${{ steps.get_version.outputs.RELEASE_VERSION }}
+          body: |
+            ## Changelog
+            
+            ##${{ steps.get_changelog.outputs.CHANGELOG }}
+          draft: false
+          prerelease: ${{ endsWith(steps.get_version.outputs.RELEASE_VERSION, '-beta') }}
+
+      - name: Upload Windows Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}.exe
+          asset_name: Tautulli-windows-${{ steps.get_version.outputs.RELEASE_VERSION }}.exe
+          asset_content_type: application/vnd.microsoft.portable-executable
+
+      - name: Upload MacOS Installer
+        if: env.WORKFLOW_CONCLUSION == 'success'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}.pkg
+          asset_name: Tautulli-macos-${{ steps.get_version.outputs.RELEASE_VERSION }}.pkg
+          asset_content_type: application/vnd.apple.installer+xml

.github/workflows/publishdocker-branch.yml

@@ -1,30 +0,0 @@
-name: Publish Docker Branch
-on:
-  push:
-    branches: [master, beta, nightly]
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@master
-      - name: Get Branch
-        run: echo ::set-env name=BRANCH::${GITHUB_REF#refs/heads/}
-      - name: Publish to Registry
-        uses: elgohr/Publish-Docker-Github-Action@master
-        env:
-          VERSION: ${{ github.sha }}
-        with:
-          name: tautulli/tautulli
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          dockerfile: Dockerfile
-          buildargs: VERSION, BRANCH
-      - name: Post Status to Discord
-        uses: sarisia/actions-status-discord@v1
-        if: always()
-        with:
-          webhook: ${{ secrets.DISCORD_WEBHOOK }}
-          status: ${{ job.status }}
-          job: ${{ github.workflow }}
-          nofail: true

.github/workflows/publishdocker-release.yml

@@ -1,34 +0,0 @@
-name: Publish Docker Release
-on:
-  push:
-    tags:
-      - 'v*'
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@master
-      - name: Get Branch
-        run: echo ::set-env name=BRANCH::${GITHUB_REF#refs/heads/}
-      - name: Get Release Version
-        run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF/refs\/tags\//}
-      - name: Publish to Registry
-        uses: elgohr/Publish-Docker-Github-Action@master
-        env:
-          VERSION: ${{ github.sha }}
-        with:
-          name: tautulli/tautulli
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          dockerfile: Dockerfile
-          buildargs: VERSION, BRANCH
-          tags: ${{ env.RELEASE_VERSION }}
-      - name: Post Status to Discord
-        uses: sarisia/actions-status-discord@v1
-        if: always()
-        with:
-          webhook: ${{ secrets.DISCORD_WEBHOOK }}
-          status: ${{ job.status }}
-          job: ${{ github.workflow }}
-          nofail: true

.github/workflows/publishrelease-beta.yml

@@ -1,29 +0,0 @@
-name: Create Pre-Release
-on:
-  push:
-    tags:
-      - 'v*-beta'
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@master
-      - name: Get Release Version
-        run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF/refs\/tags\//}
-      - name: Get Changelog
-        run: echo ::set-env name=CHANGELOG::"$( sed -n '/^## /{p; :loop n; p; /^## /q; b loop}' CHANGELOG.md | sed '$d' | sed '$d' | sed '$d' | sed ':a;N;$!ba;s/\n/%0A/g' )"
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ env.RELEASE_VERSION }}
-          release_name: Tautulli ${{ env.RELEASE_VERSION }}
-          body: |
-            ## Changelog
-            
-            ##${{ env.CHANGELOG }}
-          draft: false
-          prerelease: true

.github/workflows/publishrelease-master.yml

@@ -1,30 +0,0 @@
-name: Create Release
-on:
-  push:
-    tags:
-      - 'v*'
-      - '!v*-beta'
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@master
-      - name: Get Release Version
-        run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF/refs\/tags\//}
-      - name: Get Changelog
-        run: echo ::set-env name=CHANGELOG::"$( sed -n '/^## /{p; :loop n; p; /^## /q; b loop}' CHANGELOG.md | sed '$d' | sed '$d' | sed '$d' | sed ':a;N;$!ba;s/\n/%0A/g' )"
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ env.RELEASE_VERSION }}
-          release_name: Tautulli ${{ env.RELEASE_VERSION }}
-          body: |
-            ## Changelog
-            
-            ##${{ env.CHANGELOG }}
-          draft: false
-          prerelease: false

.gitignore

@@ -19,6 +19,8 @@ backups/*
 cache/*
 newsletters/*
 *.mmdb
+version.txt
+branch.txt
 
 # HTTPS Cert/Key #
 ##################
@@ -74,3 +76,7 @@ _ReSharper*/
 /logs
 .project
 .pydevproject
+
+#Ignore files generated by pyinstaller
+/build
+/dist

API.md

@@ -88,7 +88,8 @@ Required parameters:
     section_id (str):       The id of the Plex library section
 
 Optional parameters:
-    None
+    server_id (str):        The Plex server identifier of the library section
+    row_ids (str):          Comma separated row ids to delete, e.g. "2,3,8"
 
 Returns:
     None
@@ -103,7 +104,7 @@ Required parameters:
     user_id (str):          The id of the Plex user
 
 Optional parameters:
-    None
+    row_ids (str):          Comma separated row ids to delete, e.g. "2,3,8"
 
 Returns:
     None
@@ -114,6 +115,21 @@ Returns:
 Delete and recreate the cache directory.
 
 
+### delete_history
+Delete history rows from Tautulli.
+
+```
+Required parameters:
+    row_ids (str):          Comma separated row ids to delete, e.g. "65,110,2,3645"
+
+Optional parameters:
+    None
+
+Returns:
+    None
+```
+
+
 ### delete_hosted_images
 Delete the images uploaded to image hosting services.
 
@@ -146,7 +162,8 @@ Required parameters:
     section_id (str):       The id of the Plex library section
 
 Optional parameters:
-    None
+    server_id (str):        The Plex server identifier of the library section
+    row_ids (str):          Comma separated row ids to delete, e.g. "2,3,8"
 
 Returns:
     None
@@ -173,10 +190,13 @@ Delete the 3rd party API lookup info.
 
 ```
 Required parameters:
+    None
+
+Optional parameters:
     rating_key (int):       1234
                             (Note: Must be the movie, show, artist, album, or track rating key)
-Optional parameters:
-    None
+    service (str):          'themoviedb' or 'tvmaze' or 'musicbrainz'
+    delete_all (bool):      'true' to delete all images form the service
 
 Returns:
     json:
@@ -275,6 +295,10 @@ Returns:
 ```
 
 
+### delete_recently_added
+Flush out all of the recently added items in the database.
+
+
 ### delete_temp_sessions
 Flush out all of the temporary sessions in the database.
 
@@ -287,7 +311,7 @@ Required parameters:
     user_id (str):          The id of the Plex user
 
 Optional parameters:
-    None
+    row_ids (str):          Comma separated row ids to delete, e.g. "2,3,8"
 
 Returns:
     None
@@ -327,6 +351,7 @@ Required parameters:
 
 Optional parameters:
     custom_thumb (str):         The URL for the custom library thumbnail
+    custom_art (str):           The URL for the custom library background art
     keep_history (int):         0 or 1
 
 Returns:
@@ -647,7 +672,7 @@ Returns:
 
 
 ### get_geoip_lookup
-Get the geolocation info for an IP address. The GeoLite2 database must be installed.
+Get the geolocation info for an IP address.
 
 ```
 Required parameters:
@@ -658,7 +683,7 @@ Optional parameters:
 
 Returns:
     json:
-        {"continent": "North America",
+        {"code": 'US",
          "country": "United States",
          "region": "California",
          "city": "Mountain View",
@@ -668,9 +693,6 @@ Returns:
          "longitude": -122.0838,
          "accuracy": 1000
          }
-    json:
-        {"error": "The address 127.0.0.1 is not in the database."
-         }
 ```
 
 
@@ -718,7 +740,6 @@ Returns:
               "group_count": 1,
               "group_ids": "1124",
               "guid": "com.plexapp.agents.thetvdb://121361/6/1?lang=en",
-              "id": 1124,
               "ip_address": "xxx.xxx.xxx.xxx",
               "live": 0,
               "media_index": 17,
@@ -734,6 +755,7 @@ Returns:
               "player": "Castle-PC",
               "rating_key": 4348,
               "reference_id": 1123,
+              "row_id": 1124,
               "session_key": null,
               "started": 1462688107,
               "state": null,
@@ -845,6 +867,7 @@ Returns:
         [{"art": "/:/resources/show-fanart.jpg",
           "child_count": "3745",
           "count": "62",
+          "is_active": 1,
           "parent_count": "240",
           "section_id": "2",
           "section_name": "TV Shows",
@@ -886,7 +909,8 @@ Returns:
               "do_notify_created": "Checked",
               "duration": 1578037,
               "guid": "com.plexapp.agents.thetvdb://121361/6/1?lang=en",
-              "id": 1128,
+              "histroy_row_id": 1128,
+              "is_active": 1,
               "keep_history": "Checked",
               "labels": [],
               "last_accessed": 1462693216,
@@ -902,9 +926,11 @@ Returns:
               "parent_title": "",
               "plays": 772,
               "rating_key": 153037,
+              "row_id": 1,
               "section_id": 2,
               "section_name": "TV Shows",
               "section_type": "Show",
+              "server_id": "ds48g4r354a8v9byrrtr697g3g79w",
               "thumb": "/library/metadata/153036/thumb/1462175062",
               "year": 2016
               },
@@ -932,13 +958,16 @@ Returns:
          "deleted_section": 0,
          "do_notify": 1,
          "do_notify_created": 1,
+         "is_active": 1,
          "keep_history": 1,
          "library_art": "/:/resources/movie-fanart.jpg",
          "library_thumb": "/:/resources/movie.png",
          "parent_count": null,
+         "row_id": 1,
          "section_id": 1,
          "section_name": "Movies",
-         "section_type": "movie"
+         "section_type": "movie",
+         "server_id": "ds48g4r354a8v9byrrtr697g3g79w"
          }
 ```
 
@@ -1054,10 +1083,11 @@ Get a library's watch time statistics.
 
 ```
 Required parameters:
-    section_id (str):               The id of the Plex library section
+    section_id (str):       The id of the Plex library section
 
 Optional parameters:
     grouping (int):         0 or 1
+    query_days (str):       Comma separated days, e.g. "1,7,30,0"
 
 Returns:
     json:
@@ -2214,10 +2244,13 @@ Returns:
          "do_notify": 1,
          "email": "Jon.Snow.1337@CastleBlack.com",
          "friendly_name": "Jon Snow",
+         "is_active": 1,
+         "is_admin": 0,
          "is_allow_sync": 1,
          "is_home_user": 1,
          "is_restricted": 0,
          "keep_history": 1,
+         "row_id": 1,
          "shared_libraries": ["10", "1", "4", "5", "15", "20", "2"],
          "user_id": 133788,
          "user_thumb": "https://plex.tv/users/k10w42309cynaopq/avatar",
@@ -2370,6 +2403,7 @@ Required parameters:
 
 Optional parameters:
     grouping (int):         0 or 1
+    query_days (str):       Comma separated days, e.g. "1,7,30,0"
 
 Returns:
     json:
@@ -2413,11 +2447,13 @@ Returns:
           "filter_music": "",
           "filter_photos": "",
           "filter_tv": "",
+          "is_active": 1,
           "is_admin": 0,
           "is_allow_sync": 1,
           "is_home_user": 1,
           "is_restricted": 0,
           "keep_history": 1,
+          "row_id": 1,
           "server_token": "PU9cMuQZxJKFBtGqHk68",
           "shared_libraries": "1;2;3",
           "thumb": "https://plex.tv/users/k10w42309cynaopq/avatar",
@@ -2457,8 +2493,9 @@ Returns:
               "duration": 2998290,
               "friendly_name": "Jon Snow",
               "guid": "com.plexapp.agents.thetvdb://121361/6/1?lang=en",
-              "id": 1121,
+              "history_row_id": 1121,
               "ip_address": "xxx.xxx.xxx.xxx",
+              "is_active": 1,
               "keep_history": "Checked",
               "last_played": "Game of Thrones - The Red Woman",
               "last_seen": 1462591869,
@@ -2472,6 +2509,7 @@ Returns:
               "player": "Plex Web (Chrome)",
               "plays": 487,
               "rating_key": 153037,
+              "row_id": 1,
               "thumb": "/library/metadata/153036/thumb/1462175062",
               "transcode_decision": "transcode",
               "user_id": 133788,
@@ -2517,26 +2555,27 @@ Returns:
 
 
 ### import_database
-Import a PlexWatch or Plexivity database into Tautulli.
+Import a Tautulli, PlexWatch, or Plexivity database into Tautulli.
 
 ```
 Required parameters:
-    app (str):                      "plexwatch" or "plexivity"
+    app (str):                      "tautulli" or "plexwatch" or "plexivity"
     database_path (str):            The full path to the plexwatch database file
-    table_name (str):               "processed" or "grouped"
+    method (str):                   For Tautulli only, "merge" or "overwrite"
+    table_name (str):               For PlexWatch or Plexivity only, "processed" or "grouped"
+
 
 Optional parameters:
-    import_ignore_interval (int):   The minimum number of seconds for a stream to import
+    backup (bool):                  For Tautulli only, true or false whether to backup
+                                    the current database before importing
+    import_ignore_interval (int):   For PlexWatch or Plexivity only, the minimum number
+                                    of seconds for a stream to import
 
 Returns:
     None
 ```
 
 
-### install_geoip_db
-Downloads and installs the GeoLite2 database
-
-
 ### notify
 Send a notification using Tautulli.
 
@@ -2733,7 +2772,7 @@ Returns:
 ### sql
 Query the Tautulli database with raw SQL. Automatically makes a backup of
 the database if the latest backup is older then 24h. `api_sql` must be
-manually enabled in the config file.
+manually enabled in the config file while Tautulli is shut down.
 
 ```
 Required parameters:
@@ -2813,10 +2852,6 @@ Returns:
 ```
 
 
-### uninstall_geoip_db
-Uninstalls the GeoLite2 database
-
-
 ### update
 Update Tautulli.
 

CHANGELOG.md

@@ -1,10 +1,112 @@
 # Changelog
 
-## v2.2.0-beta (2020-02-27)
+## v2.5.2 (2020-07-01)
+
+* Announcements:
+  * Tautulli now supports Python 3!
+  * Python 2 is still supported for the time being, but it is recommended to upgrade to Python 3.
+* Notifications:
+  * Fix: Error uploading images to Cloudinary on Python 2.
+  * Fix: Testing browser notifications alert not disappearing.
+  * Change: Default recently added notification delay set to 300 seconds.
+* UI: 
+  * Fix: MacOS menu bar icon causing Tautulli to fail to start.
+  * Fix: Unable to login to Tautulli on Python 2.
+  * New: Windows and MacOS setting to enable Tautulli to start automatically when you login.
+  * New: Added menu bar icon for MacOS.
+  * New: Ability to import a Tautulli database in the settings.
+  * New: Added Tautulli news area on the settings page.
+  * New: Added platform icon for LG devices.
+  * Remove: Ability to login to Tautulli using a Plex username and password has been removed. Login using a Plex.tv account is only supported via OAuth.
+* Mobile App:
+  * Fix: Improved API security and validation when registering the Android app.
+* Docker:
+  * Fix: Docker container not respecting the PUID and PGID environment variables.
+* Other:
+  * Fix: Error creating self-signed certificates on Python 3.
+  * Fix: Tautulli login session cookie not set on the HTTP root path.
+  * New: Windows and MacOS app installers to install Tautulli without needing Python installed.
+
+
+## v2.2.4 (2020-05-16)
+
+* Monitoring:
+  * Fix: Show "None" as the subtitle source on the activity card for user selected subtitles.
+* UI:
+  * Fix: Deleted libraries were showing up on the homepage library cards.
+  * Fix: Libraries could get stuck as inactive in the database in some instances.
+* API:
+  * Fix: Incorrect title was being returned for the get_history API command.
+* Other:
+  * Fix: Plex remote access check was not being rescheduled after changing the settings.
+
+
+## v2.2.3 (2020-05-01)
+
+* Notifications:
+  * Fix: Notification grouping by season/album and show/artist not enabled by default.
+  * Fix: The rating key notification parameter was being overwritten when 3rd party lookup was enabled.
+  * Fix: Missing artist value for Musicbrainz lookup in certain situations. 
+  * New: Added notification trigger for Tautulli database corruption.
+  * New: Added TAUTULLI_PYTHON_VERSION to script notification environment variables.
+  * New: Added Plex Android / iOS App notification agent.
+  * New: Added bandwidth notification parameters.
+  * New: Added user thumb to notification parameters.
+  * New: Added initial stream notification parameter and threshold setting to determine if a stream is the first stream of a continuous streaming session.
+  * New: Added Plex remote access notification parameters.
+  * Change: The file size notification parameter is now reported in SI units. (Thanks @aaronldunlap)
+* UI:
+  * Fix: Delete lookup info from the media info page failing.
+  * Fix: XBMC platform icon not being redirected to the Kodi platform icon.
+  * Fix: History table was not being refreshed after deleting entries.
+  * New: Added icon on the users table to indicate if the user is not on the Plex server.
+  * New: Added icon on the libraries table to indicate if the library is not on the Plex server.
+  * Change: Improved deleting libraries so libraries with the same section ID are not also deleted.
+* Mobile App:
+  * Fix: Temporary device token was not being invalidated after cancelling device registration.
+* API:
+  * Fix: Returning XML from the API failing due to unicode characters.
+  * Fix: Grouping parameter for various API commands not falling back to default setting.
+  * New: Added time_queries parameter to get_library_watch_time_stats and get_user_watch_time_stats API command. (Thanks @KaasKop97)
+  * New: Added an "is_active" return value to the get_user, get_users, get_library, and get_libraries API commands which indicates if the user or library is on the Plex server.
+  * New: Added delete_history API command.
+  * Change: Added optional parameter for row_ids for delete_library, delete_user, delete_all_library_history, and delete_all_user_history API commands.
+* Other:
+  * Fix: Update failing on CentOS due to an older git version.
+  * Fix: Manifest file for creating a web app had incorrect info.
+  * Fix: Auto-updater was not scheduled when enabling the setting unless Tautulli was restarted.
+  * New: Docker images updated to support ARM platforms.
+  * Change: Remove the unnecessary optional Plex logs volume from the Docker image.
+  * Change: Use Plex.tv for GeoIP lookup instead of requiring the MaxMind GeoLite2 database.
+
+
+## v2.2.1 (2020-03-28)
+
+* Notifications:
+  * Fix: File size notification parameter incorrectly truncated to an integer.
+  * Fix: Notification grouping by season/album not enabled by default.
+  * New: Added transcode decision counts to notification parameters.
+  * Change: Tags (<>) are no longer stripped from from Webhook notification text.
+* Newsletter:
+  * New: Added favicon to newsletter template when viewing as a web page.
+* UI:
+  * Fix: Username missing from the Synced Items table.
+  * Fix: Windows system tray icon not enabled by default.
+  * Fix: Saving a mobile device with a blank friendly name caused an error.
+  * New: Added IMDb and Rotten Tomato Ratings to info pages.
+  * New: Added button in settings to delete all 3rd party metadata lookup info in the database.
+  * New: Added button in settings to flush recently added items in the database.
+* API:
+  * New: Added delete_recenly_added API command to flush recently added items.
+  * Change: Updated delete_lookup_info API command parameters to allow deleteing all 3rd party metadata lookup info.
+
+
+## v2.2.0 (2020-03-08)
 
 * Important Note!
   * All Live TV changes requires Plex Media Server 1.18.7 or higher. 
 * Monitoring:
+  * Fix: Improved IPv6 display on the activity cards. (Thanks @felixbuenemann)
   * New: Added Live TV metadata and posters to the activity cards.
   * Change: Show bandwidth in Gbps when greater than 1000 Mbps.
 * History:
@@ -18,8 +120,17 @@
 * Notifications:
   * Fix: Race condition causing stream count to be incorrect for playback stop notifications.
   * New: Added Live TV channel notification parameters.
+  * New: Added Plex background art notification parameter.
+    * Note: This is the Plex API endpoint to retrieve the background art, not the actual image.
+  * New: Added poster images for clip notifications.
+  * Change: Default Webhook notification method to POST.
+* UI:
+  * Fix: Windows platform showing up twice on the Most Active Platforms statistics card.
+  * New: Added option to change the background art for library sections when editing a library.
+  * New: Added button to reset Tautulli git installation in settings to fix failed git updates.
 * API:
   * New: Added ability to filter history using a "live" media type and by guid for the get_history API command.
+  * New: Added cutsom_art parameter to the edit_library API command.
 * Other:
   * Change: Add crossorigin use-credentials attribute to manifest tags. (Thanks @pkoenig10)
   * Change: Disable automatic updates for Docker containers. Updates are now handled by updating the Docker container.

CONTRIBUTING.md

@@ -9,7 +9,7 @@ All pull requests should be based on the `nightly` branch, to minimize cross mer
 ### Python Code
 
 #### Compatibility
-The code should work with Python 2.7. Note that Tautulli runs on many different platforms.
+The code should work with Python 2.7.17 or Python 3.6+. Note that Tautulli runs on many different platforms.
 
 Re-use existing code. Do not hesitate to add logging in your code. You can the logger module `plexpy.logger.*` for this. Web requests are invoked via `plexpy.request.*` and derived ones. Use these methods to automatically add proper and meaningful error handling.
 
@@ -38,4 +38,4 @@ HTML5 compatible browsers are targeted.
 * 4 space indentation
 * `methodName`
 * `variableName`
-* `ClassName`
+* `ClassName`

Dockerfile

@@ -1,9 +1,9 @@
-FROM python:2.7.17-slim
+FROM tautulli/tautulli-baseimage:python3
 
-LABEL maintainer="TheMeanCanEHdian"
+LABEL maintainer="Tautulli"
 
-ARG VERSION
 ARG BRANCH
+ARG COMMIT
 
 ENV TAUTULLI_DOCKER=True
 ENV TZ=UTC
@@ -11,21 +11,13 @@ ENV TZ=UTC
 WORKDIR /app
 
 RUN \
-apt-get -q -y update --no-install-recommends && \
-apt-get install -q -y --no-install-recommends \
-  curl && \
-rm -rf /var/lib/apt/lists/* && \
-pip install --no-cache-dir --upgrade pip && \
-pip install --no-cache-dir --upgrade \
-  pycryptodomex \
-  pyopenssl && \
-echo ${VERSION} > /app/version.txt && \
-echo ${BRANCH} > /app/branch.txt
+  echo ${BRANCH} > /app/branch.txt && \
+  echo ${COMMIT} > /app/version.txt
 
 COPY . /app
 
-CMD [ "python", "Tautulli.py", "--datadir", "/config" ]
+ENTRYPOINT [ "./start.sh" ]
 
-VOLUME /config /plex_logs
+VOLUME /config
 EXPOSE 8181
 HEALTHCHECK  --start-period=90s CMD curl -ILfSs http://localhost:8181/status > /dev/null || curl -ILfkSs https://localhost:8181/status > /dev/null || exit 1

PlexPy.py

@@ -1,8 +1,4 @@
-#!/bin/sh
-''''which python    >/dev/null 2>&1 && exec python    "$0" "$@" # '''
-''''which python2   >/dev/null 2>&1 && exec python2   "$0" "$@" # '''
-''''which python2.7 >/dev/null 2>&1 && exec python2.7 "$0" "$@" # '''
-''''exec echo "Error: Python not found!" # '''
+#!/usr/bin/env python
 
 # -*- coding: utf-8 -*-
 

README.md

@@ -29,14 +29,15 @@ This project is based on code from [Headphones](https://github.com/rembo10/headp
 
 ## Installation & Support
 
-[![Python](https://img.shields.io/badge/python-v2.7.17-blue?style=flat-square)](https://python.org/downloads/release/python-2717/)
+[![Python](https://img.shields.io/badge/python-2.7.17,%203.6,%203.7,%203.8-blue?style=flat-square)](https://python.org/downloads)
 [![Docker Pulls](https://img.shields.io/docker/pulls/tautulli/tautulli?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli)
 [![Docker Stars](https://img.shields.io/docker/stars/tautulli/tautulli?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli)
 
 | Status | Branch: `master` | Branch: `beta` | Branch: `nightly` |
 | --- | --- | --- | --- |
-| Release | [![Release@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![Release Date@master](https://img.shields.io/github/release-date/Tautulli/Tautulli?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/releases/latest) | [![Release@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![Commits@nightly](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/beta?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/beta) | [![Last Commits@nightly](https://img.shields.io/github/last-commit/Tautulli/Tautulli/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) <br> [![Commits@nightly](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) |
-| Docker  | [![Docker@master](https://img.shields.io/badge/tautulli-tautulli:latest-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker%20Branch/master?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=branch%3Amaster) | [![Docker@beta](https://img.shields.io/badge/tautulli-tautulli:beta-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@beta](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker%20Branch/beta?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=branch%3Abeta) | [![Docker@nightly](https://img.shields.io/badge/tautulli-tautulli:nightly-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@nightly](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker%20Branch/nightly?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=branch%3Anightly) |
+| Release   | [![Release@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![Release Date@master](https://img.shields.io/github/release-date/Tautulli/Tautulli?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/releases/latest) | [![Release@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![Commits@beta](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/beta?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/beta) | [![Last Commits@nightly](https://img.shields.io/github/last-commit/Tautulli/Tautulli/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) <br> [![Commits@nightly](https://img.shields.io/github/commits-since/Tautulli/Tautulli/latest/nightly?style=flat-square&color=blue)](https://github.com/Tautulli/Tautulli/commits/nightly) |
+| Docker    | [![Docker@master](https://img.shields.io/badge/docker-latest-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/master?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Amaster) | [![Docker@beta](https://img.shields.io/badge/docker-beta-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@beta](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/beta?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Abeta) | [![Docker@nightly](https://img.shields.io/badge/docker-nightly-blue?style=flat-square)](https://hub.docker.com/r/tautulli/tautulli) <br> [![Docker Build@nightly](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Docker/nightly?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Docker"+branch%3Anightly) |
+| Installer | [![Windows@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=windows&style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![MacOS@master](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=macos&style=flat-square)](https://github.com/Tautulli/Tautulli/releases/latest) <br> [![Installer Build@master](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Release/master?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Release"+branch%3Amaster) | [![Windows@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=windows&include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![MacOS@beta](https://img.shields.io/github/v/release/Tautulli/Tautulli?label=macos&include_prereleases&style=flat-square)](https://github.com/Tautulli/Tautulli/releases) <br> [![Installer Build@beta](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Release/beta?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Release"+branch%3Abeta) | [![Installer Build@nightly](https://img.shields.io/github/workflow/status/Tautulli/Tautulli/Publish%20Release/nightly?style=flat-square)](https://github.com/Tautulli/Tautulli/actions?query=workflow%3A"Publish+Release"+branch%3Anightly) |
 
 [![Wiki](https://img.shields.io/badge/github-wiki-black?style=flat-square)](https://github.com/Tautulli/Tautulli-Wiki/wiki)
 [![Discord](https://img.shields.io/discord/183396325142822912?label=discord&style=flat-square&color=7289DA)](https://tautulli.com/discord)

Tautulli.py

@@ -1,8 +1,4 @@
-#!/bin/sh
-''''which python    >/dev/null 2>&1 && exec python    "$0" "$@" # '''
-''''which python2   >/dev/null 2>&1 && exec python2   "$0" "$@" # '''
-''''which python2.7 >/dev/null 2>&1 && exec python2.7 "$0" "$@" # '''
-''''exec echo "Error: Python not found!" # '''
+#!/usr/bin/env python
 
 # -*- coding: utf-8 -*-
 
@@ -27,17 +23,24 @@ import sys
 # Ensure lib added to path, before any other imports
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib'))
 
+from future.builtins import str
+
+import appdirs
 import argparse
 import datetime
 import locale
 import pytz
 import signal
 import time
+import threading
 import tzlocal
 
 import plexpy
-from plexpy import config, database, helpers, logger, webstart
-
+from plexpy import common, config, database, helpers, logger, webstart
+if common.PLATFORM == 'Windows':
+    from plexpy import windows
+elif common.PLATFORM == 'Darwin':
+    from plexpy import macos
 
 # Register signals, such as CTRL + C
 signal.signal(signal.SIGINT, plexpy.sig_handler)
@@ -51,12 +54,14 @@ def main():
     """
 
     # Fixed paths to Tautulli
-    if hasattr(sys, 'frozen'):
+    if hasattr(sys, 'frozen') and hasattr(sys, '_MEIPASS'):
+        plexpy.FROZEN = True
         plexpy.FULL_PATH = os.path.abspath(sys.executable)
+        plexpy.PROG_DIR = sys._MEIPASS
     else:
         plexpy.FULL_PATH = os.path.abspath(__file__)
+        plexpy.PROG_DIR = os.path.dirname(plexpy.FULL_PATH)
 
-    plexpy.PROG_DIR = os.path.dirname(plexpy.FULL_PATH)
     plexpy.ARGS = sys.argv[1:]
 
     # From sickbeard
@@ -122,7 +127,7 @@ def main():
 
     if args.dev:
         plexpy.DEV = True
-        logger.debug(u"Tautulli is running in the dev environment.")
+        logger.debug("Tautulli is running in the dev environment.")
 
     if args.daemon:
         if sys.platform == 'win32':
@@ -175,6 +180,8 @@ def main():
     # Determine which data directory and config file to use
     if args.datadir:
         plexpy.DATA_DIR = args.datadir
+    elif plexpy.FROZEN:
+        plexpy.DATA_DIR = appdirs.user_data_dir("Tautulli", False)
     else:
         plexpy.DATA_DIR = plexpy.PROG_DIR
 
@@ -229,25 +236,48 @@ def main():
         try:
             import OpenSSL
         except ImportError:
-            logger.warn("The pyOpenSSL module is missing. Install this " \
+            logger.warn("The pyOpenSSL module is missing. Install this "
                         "module to enable HTTPS. HTTPS will be disabled.")
             plexpy.CONFIG.ENABLE_HTTPS = False
 
     # Try to start the server. Will exit here is address is already in use.
     webstart.start()
 
-    # Windows system tray icon
-    if os.name == 'nt' and plexpy.CONFIG.WIN_SYS_TRAY:
-        plexpy.win_system_tray()
-
-    logger.info("Tautulli is ready!")
+    if common.PLATFORM == 'Windows':
+        if plexpy.CONFIG.SYS_TRAY_ICON:
+            plexpy.WIN_SYS_TRAY_ICON = windows.WindowsSystemTray()
+            plexpy.WIN_SYS_TRAY_ICON.start()
+        windows.set_startup()
+    elif common.PLATFORM == 'Darwin':
+        macos.set_startup()
 
     # Open webbrowser
     if plexpy.CONFIG.LAUNCH_BROWSER and not args.nolaunch and not plexpy.DEV:
         plexpy.launch_browser(plexpy.CONFIG.HTTP_HOST, plexpy.HTTP_PORT,
                               plexpy.HTTP_ROOT)
 
-    # Wait endlessy for a signal to happen
+    if common.PLATFORM == 'Darwin' and plexpy.CONFIG.SYS_TRAY_ICON:
+        if not macos.HAS_PYOBJC:
+            logger.warn("The pyobjc module is missing. Install this "
+                        "module to enable the MacOS menu bar icon.")
+            plexpy.CONFIG.SYS_TRAY_ICON = False
+
+        if plexpy.CONFIG.SYS_TRAY_ICON:
+            # MacOS menu bar icon must be run on the main thread and is blocking
+            # Start the rest of Tautulli on a new thread
+            threading.Thread(target=wait).start()
+            plexpy.MAC_SYS_TRAY_ICON = macos.MacOSSystemTray()
+            plexpy.MAC_SYS_TRAY_ICON.start()
+        else:
+            wait()
+    else:
+        wait()
+
+
+def wait():
+    logger.info("Tautulli is ready!")
+
+    # Wait endlessly for a signal to happen
     while True:
         if not plexpy.SIGNAL:
             try:
@@ -263,11 +293,16 @@ def main():
                 plexpy.shutdown(restart=True)
             elif plexpy.SIGNAL == 'checkout':
                 plexpy.shutdown(restart=True, checkout=True)
-            else:
+            elif plexpy.SIGNAL == 'reset':
+                plexpy.shutdown(restart=True, reset=True)
+            elif plexpy.SIGNAL == 'update':
                 plexpy.shutdown(restart=True, update=True)
+            else:
+                logger.error('Unknown signal. Shutting down...')
+                plexpy.shutdown()
 
             plexpy.SIGNAL = None
 
-# Call main()
+
 if __name__ == "__main__":
     main()

data/interfaces/default/app_import.html

@@ -5,46 +5,97 @@
             <h4 class="modal-title">Import ${app} Database</h4>
         </div>
         <div class="modal-body" id="modal-text">
-            <p class="help-block">
-                <%
-                    v = ''
-                    if app == 'PlexWatch':
-                        v = '0.3.2'
-                    elif app == 'Plexivity':
-                        v = '0.9.8'
-                %>
-                <strong>Please ensure your ${app} database is at version ${v} or higher.</strong>
-            </p>
-            <div class="form-group">
-                <label for="db_location">Database Location</label>
-                <div class="row">
-                    <div class="col-xs-8">
-                        <input type="text" class="form-control" id="db_location" name="db_location" value="" required>
+            <form id="import_database" enctype="multipart/form-data" method="post" name="import_database">
+                <input type="hidden" id="import_app" name="import_app" value="${app.lower()}" />
+                % if app in ('PlexWatch', 'Plexivity'):
+                <p class="help-block">
+                    <%
+                        v = ''
+                        if app == 'PlexWatch':
+                            v = '0.3.2'
+                        elif app == 'Plexivity':
+                            v = '0.9.8'
+                    %>
+                    <strong>Please ensure your ${app} database is at version ${v} or higher.</strong>
+                </p>
+                % endif
+                <div class="form-group">
+                    <label for="import_database_file">Option 1: Upload a Database File</label>
+                    <div class="row">
+                        <div class="col-xs-12">
+                            <div class="input-group">
+                                <label for="import_database_file" class="input-group-btn">
+                                    <span class="btn btn-form">Upload</span>
+                                    <input type="file" style="display: none;" id="import_database_file" name="import_database_file" required>
+                                </label>
+                                <input id="import_database_file_name" type="text" class="form-control" disabled>
+                            </div>
+                        </div>
                     </div>
+                    <p class="help-block">Upload the ${app} database you wish to import.</p>
                 </div>
-                <p class="help-block">Enter the path and file name for the ${app} database you wish to import.</p>
-            </div>
-            <div class="form-group">
-                <label for="table_name">Table Name</label>
-                <div class="row">
-                    <div class="col-xs-4">
-                        <select id="table_name" class="form-control" name="table_name">
-                            <option value="processed">processed</option>
-                            <option value="grouped">grouped</option>
-                        </select>
+                <div class="form-group">
+                    <label for="import_database_path">Option 2: Browse for a Database File</label>
+                    <div class="row">
+                        <div class="col-xs-12">
+                            <div class="input-group">
+                                <span class="input-group-btn">
+                                    <button class="btn btn-form" type="button" id="import_database_path_browse">Browse</button>
+                                </span>
+                                <input type="text" class="form-control" id="import_database_path" name="import_database_path" value="" required disabled>
+                            </div>
+                        </div>
                     </div>
+                    <p class="help-block">Browse for the ${app} database you wish to import.</p>
                 </div>
-                <p class="help-block">The table name from which you wish to import. Only import one of these, importing both will result in duplicated data.</p>
-            </div>
-            <div class="form-group">
-                <label for="import_ignore_interval">Ignore Interval</label>
-                <div class="row">
-                    <div class="col-xs-2">
-                        <input type="text" class="form-control" id="import_ignore_interval" name="import_ignore_interval" value="120" required>
+                % if app == 'Tautulli':
+                <div class="form-group">
+                    <label for="table_name">Import Method</label>
+                    <div class="row">
+                        <div class="col-xs-4">
+                            <select class="form-control" id="import_method" name="import_method">
+                                <option value="merge">Merge</option>
+                                <option value="overwrite">Overwrite</option>
+                            </select>
+                        </div>
                     </div>
+                    <p class="help-block">Select how you would like to import the Tautulli history.</p>
+                    <ul class="help-block" style="padding-inline-start: 15px;">
+                        <li><strong>Merge</strong> will add all history and remove any duplicates from the imported database into the current database.</li>
+                        <li><strong>Overwrite</strong> will replace all history in the current database with the imported database.</li>
+                    </ul>
+                    <p class="help-block">Note: Libraries, users, notification agents, newsletter agents, and registered mobile devices will also be imported</p>
                 </div>
-                <p class="help-block">Enter the minimum duration (in seconds) an item must have been active for. Set to 0 to import all.</p>
-            </div>
+                <div class="checkbox">
+                    <label>
+                        <input type="checkbox" name="import_backup_db" id="import_backup_db" value="1" checked> Backup Current Database
+                    </label>
+                    <p class="help-block">Automatically create a backup of the current database before importing.</p>
+                </div>
+                % else:
+                <div class="form-group">
+                    <label for="import_table_name">Table Name</label>
+                    <div class="row">
+                        <div class="col-xs-4">
+                            <select class="form-control" id="import_table_name" name="import_table_name">
+                                <option value="processed">Processed</option>
+                                <option value="grouped">Grouped</option>
+                            </select>
+                        </div>
+                    </div>
+                    <p class="help-block">Select the table name from which you wish to import. Only import one of these, importing both will result in duplicated data.</p>
+                </div>
+                <div class="form-group">
+                    <label for="import_ignore_interval">Ignore Interval</label>
+                    <div class="row">
+                        <div class="col-xs-2">
+                            <input type="text" class="form-control" id="import_ignore_interval" name="import_ignore_interval" value="120" required>
+                        </div>
+                    </div>
+                    <p class="help-block">Enter the minimum duration (in seconds) an item must have been active for. Set to 0 to import all.</p>
+                </div>
+                % endif
+            </form>
         </div>
         <div class="modal-footer">
             <div>
@@ -55,24 +106,97 @@
     </div>
 </div>
 <script>
-    // Send database path to import script
+    $('#import_database_path_browse').click(function () {
+        $('#browse-path-type').text('Databse File');
+        $('#browse-path-modal').modal('show');
+        browsePath(null, null, '.db');
+    });
+    $('#select-browse-file').click(function () {
+        $('#browse-path-modal').modal('hide');
+        $("#import_database_path").val($('#browse-path').val());
+    });
+
+    $('#import_database_file').change(function() {
+        if ($(this)[0].files[0]) {
+            $('#import_database_file_name').val($(this)[0].files[0].name);
+        }
+    });
+
     $("#import_db").click(function() {
-        var database_path = $("#db_location").val();
-        var table_name = $("#table_name").val();
-        var import_ignore_interval = $("#import_ignore_interval").val();
+        $(this).prop('disabled', true);
+
+        var app = $("#import_app").val();
+        var database_file = $('#import_database_file')[0].files[0];
+        var database_path = $("#import_database_path").val();
+        var method = $("#import_method").val();
+        var backup = $("#import_backup_db").is(':checked');
+        var table_name = $("#import_table_name").val();
+        var ignore_interval = $("#import_ignore_interval").val();
+
+        var content_type;
+        var process_data;
+        var data;
+
+        if (database_file) {
+            content_type = false;
+            process_data = false;
+            data = new FormData();
+            data.append('app', app);
+            data.append('database_file', database_file);
+            data.append('method', method);
+            data.append('backup', backup);
+            data.append('table_name', table_name);
+            data.append('ignore_interval', ignore_interval);
+        } else {
+            content_type = 'application/x-www-form-urlencoded; charset=UTF-8';
+            process_data = true;
+            data = {
+                app: app,
+                database_path: database_path,
+                method: method,
+                backup: backup,
+                table_name: table_name,
+                ignore_interval: ignore_interval
+            }
+        }
+
+        if (database_file) {
+            $("#status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>&nbsp; Uploading database file...');
+        } else {
+            $("#status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>');
+        }
+
         $.ajax({
             url: 'import_database',
-            data: {
-                app: "${app}",
-                database_path: database_path,
-                table_name: table_name,
-                import_ignore_interval: import_ignore_interval
-            },
+            type: 'POST',
+            data: data,
             cache: false,
             async: true,
+            contentType: content_type,
+            processData: process_data,
             success: function(data) {
-                $("#status-message").html(data);
-                $("#db_location").val('')
+                var msg;
+                if (data.result === 'success') {
+                    msg = "<i class='fa fa-check'></i>&nbsp; " + data.message;
+                } else {
+                    msg = "<i class='fa fa-exclamation-triangle'></i>&nbsp; " + data.message;
+                }
+                $("#status-message").html(msg);
+                $("#import_database_file").val(null);
+                $("#import_database_file_name").val('');
+                $("#import_database_path").val('');
+            },
+            error: function (xhr) {
+                var msg = "<i class='fa fa-exclamation-triangle'></i>&nbsp; Error (" + xhr.status + "): ";
+                if (xhr.status === 413) {
+                    msg += "file is too large to upload"
+                } else {
+                    msg += 'try again'
+                }
+                $("#status-message").html(msg);
+            },
+            complete: function(xhr) {
+                $("#import_db").prop('disabled', false);
             }
         });
     });

data/interfaces/default/base.html

@@ -55,8 +55,10 @@
         newer version</a> of Tautulli is available!<br />
         You are ${plexpy.COMMITS_BEHIND} commit${'s' if plexpy.COMMITS_BEHIND > 1 else ''} behind.<br />
         % endif
-        % if plexpy.DOCKER:
+        % if plexpy.INSTALL_TYPE == 'docker':
         Update your Docker container or <a href="#" id="updateDismiss">Dismiss</a>
+        % elif plexpy.INSTALL_TYPE in ('windows', 'macos'):
+        <a href="${anon_url('https://github.com/%s/%s/releases/tag/%s' % (plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO, plexpy.LATEST_RELEASE))}" target="_blank">Download</a> and install the latest version or <a href="#" id="updateDismiss">Dismiss</a>
         % else:
         <a href="update">Update</a> or <a href="#" id="updateDismiss">Dismiss</a>
         % endif
@@ -228,20 +230,12 @@ ${next.modalIncludes()}
                     </div>
                 </div>
                 <ul id="donation_type" class="nav nav-pills" role="tablist" style="display: flex; justify-content: center; margin: 10px 0;">
-                    <li class="active"><a href="#patreon-donation" role="tab" data-toggle="tab">Patreon</a></li>
-                    <li><a href="#github-donation" role="tab" data-toggle="tab">GitHub</a></li>
+                    <li class="active"><a href="#github-donation" role="tab" data-toggle="tab">GitHub</a></li>
+                    <li><a href="#patreon-donation" role="tab" data-toggle="tab">Patreon</a></li>
                     <li><a href="#paypal-donation" role="tab" data-toggle="tab">PayPal</a></li>
                 </ul>
                 <div class="tab-content">
-                    <div role="tabpanel" class="tab-pane active" id="patreon-donation" style="text-align: center">
-                        <p>
-                            Click the button below to continue to Patreon.
-                        </p>
-                        <a href="${anon_url('https://www.patreon.com/join/tautulli')}" target="_blank">
-                            <img src="images/become_a_patron_button.png" alt="Become a Patron" height="40">
-                        </a>
-                    </div>
-                    <div role="tabpanel" class="tab-pane" id="github-donation" style="text-align: center">
+                    <div role="tabpanel" class="tab-pane active" id="github-donation" style="text-align: center">
                         <p>
                             Click the button below to continue to GitHub.
                         </p>
@@ -249,6 +243,14 @@ ${next.modalIncludes()}
                             <i class="fa fa-heart fa-sm" style="color: #ea4aaa;"></i>&nbsp; Sponsor
                         </a>
                     </div>
+                    <div role="tabpanel" class="tab-pane" id="patreon-donation" style="text-align: center">
+                        <p>
+                            Click the button below to continue to Patreon.
+                        </p>
+                        <a href="${anon_url('https://www.patreon.com/join/tautulli')}" target="_blank">
+                            <img src="images/become_a_patron_button.png" alt="Become a Patron" height="40">
+                        </a>
+                    </div>
                     <div role="tabpanel" class="tab-pane" id="paypal-donation" style="text-align: center">
                         <p>
                             Click the button below to continue to PayPal.
@@ -291,11 +293,10 @@ ${next.modalIncludes()}
 <script src="${http_root}js/bootstrap-hover-dropdown.min.js"></script>
 <script src="${http_root}js/pnotify.custom.min.js"></script>
 <script src="${http_root}js/platform.min.js"></script>
+<script src="${http_root}js/ipaddr.min.js"></script>
 <script src="${http_root}js/script.js${cache_param}"></script>
 <script src="${http_root}js/jquery.tripleclick.min.js"></script>
-% if _session['user_group'] == 'admin' and BROWSER_NOTIFIERS:
 <script src="${http_root}js/ajaxNotifications.js"></script>
-% endif
 <script>
     % if _session['user_group'] == 'admin':
     $('body').on('click', '#updateDismiss', function() {
@@ -329,8 +330,10 @@ ${next.modalIncludes()}
                         msg = 'A <a href="' + result.compare_url + '" target="_blank">newer version</a> of Tautulli is available!<br />' +
                             'You are '+ result.commits_behind + ' commit' + (result.commits_behind > 1 ? 's' : '') + ' behind.<br />';
                     }
-                    if (result.docker) {
+                    if (result.install_type === 'docker') {
                         msg += 'Update your Docker container or <a href="#" id="updateDismiss">Dismiss</a>';
+                    } else if (result.install_type === 'windows' || result.install_type === 'macos') {
+                        msg += '<a href="' + result.release_url + '" target="_blank">Download</a> and install the latest version or <a href="#" id="updateDismiss">Dismiss</a>'
                     } else {
                         msg += '<a href="update">Update</a> or <a href="#" id="updateDismiss">Dismiss</a>';
                     }
@@ -418,6 +421,10 @@ ${next.modalIncludes()}
         $(document).on('hidden.bs.modal', '.modal', function () {
             $('.modal:visible').length && $(document.body).addClass('modal-open');
         });
+
+        % if _session['user_group'] == 'admin' and BROWSER_NOTIFIERS:
+        check_notifications();
+        % endif
     });
 
     % if _session['user_group'] != 'admin':

data/interfaces/default/css/tautulli.css

@@ -711,7 +711,6 @@ fieldset[disabled] .form-control {
     box-shadow: 0 0 4px rgba(0,0,0,.3),inset 0 0 0 1px rgba(255,255,255,.1);
 }
 .users-poster-face {
-    overflow: hidden;
     float: left;
     background-size: cover;
     background-position: center;
@@ -857,7 +856,6 @@ a .users-poster-face:hover {
     z-index: 2;
 }
 .dashboard-activity-info-platform {
-    padding: 6px !important;
     background-position: center;
     background-size: cover;
     width: 50px;
@@ -973,7 +971,7 @@ a .users-poster-face:hover {
     white-space: nowrap;
     overflow: hidden;
     text-overflow: ellipsis;
-    max-width: 140px;
+    max-width: 125px;
 }
 .dashboard-activity-info-time {
     position: absolute;
@@ -1036,13 +1034,13 @@ a .users-poster-face:hover {
 }
 .dashboard-activity-container:hover .progress-bar {
     color: rgba(255, 255, 255, 1);
-    background-image: -webkit-linear-gradient(left,rgba(0,0,0,0.25),0%,rgba(0,0,0,0),50px);
+    background-image: -webkit-linear-gradient(left,rgba(0,0,0,0.25) 0%,rgba(0,0,0,0) 50px);
     background-image: -moz-linear-gradient(left,rgba(0,0,0,0.25) 0%,rgba(0,0,0,0) 50px);
     background-image: linear-gradient(to left,rgba(0,0,0,0.25) 0%,rgba(0,0,0,0) 50px);
 }
 .dashboard-activity-container:hover .buffer-bar {
     color: rgba(255, 255, 255, 1);
-    background-image: -webkit-linear-gradient(left,rgba(0,0,0,0.25),0%,rgba(0,0,0,0),50px);
+    background-image: -webkit-linear-gradient(left,rgba(0,0,0,0.25) 0%,rgba(0,0,0,0) 50px);
     background-image: -moz-linear-gradient(left,rgba(0,0,0,0.25) 0%,rgba(0,0,0,0) 50px);
     background-image: linear-gradient(to left,rgba(0,0,0,0.25) 0%,rgba(0,0,0,0) 50px);
 }
@@ -1742,7 +1740,7 @@ a:hover .dashboard-recent-media-cover {
     top: 0;
     bottom: 0;
     background-image: -webkit-gradient(linear,left 0,left 100%,from(rgba(0,0,0,.7)),to(rgba(0,0,0,.9)));
-    background-image: -webkit-linear-gradient(top,rgba(0,0,0,.7),0,rgba(0,0,0,.9),100%);
+    background-image: -webkit-linear-gradient(top,rgba(0,0,0,.7) 0,rgba(0,0,0,.9) 100%);
     background-image: -moz-linear-gradient(top,rgba(0,0,0,.7) 0,rgba(0,0,0,.9) 100%);
     background-image: linear-gradient(to bottom,rgba(0,0,0,.7) 0,rgba(0,0,0,.9) 100%);
     background-repeat: repeat-x;
@@ -1921,6 +1919,16 @@ a:hover .summary-poster-face-track .summary-poster-face-overlay span {
     margin-left: 2px;
     color: #999;
 }
+.critic-rating {
+    display: inline-block;
+    font-size: 14px;
+    overflow: hidden;
+    white-space: nowrap;
+    margin-top: 2px;
+    height: 20px;
+    line-height: 20px;
+    float: right;
+}
 .children-list,
 .search-results-list {
     position: relative;
@@ -2153,7 +2161,7 @@ div.advanced-setting {
 li.advanced-setting {
     border-left: 1px solid #cc7b19;
 }
-.docker-setting {
+.setting-message {
     color: #cc7b19;
     margin-left: 10px;
 }
@@ -2304,6 +2312,7 @@ li.advanced-setting {
     width: 140px;
     margin-left: 10px;
     margin-bottom: 10px;
+    white-space: nowrap;
 }
 .user-player-instance-playcount h3 {
     font-size: 30px;
@@ -2986,6 +2995,9 @@ a .home-platforms-list-cover-face:hover
 .accordion li .link i.fa {
     color: #999;
 }
+.accordion li .link span.toggle-left {
+    padding-right: 5px;
+}
 .accordion li .link span.toggle-right {
     float: right;
     padding-left: 10px;
@@ -3102,12 +3114,28 @@ div.dataTables_info {
     border-left-color: #fff;
 }
 .tooltip-inner {
+    max-width: 250px;
     color: #000;
     background: #fff;
     border: 0;
     font-weight: bold;
     border-radius: 2px;
 }
+.inactive-library-tooltip,
+.inactive-user-tooltip {
+    display: inline-block;
+    position: relative;
+    width: 100%;
+    height: 100%;
+}
+.inactive-library-tooltip i.fa,
+.inactive-user-tooltip i.fa {
+    color: #E5A00D;
+    position: absolute;
+    right: 0;
+    bottom: 0;
+    text-shadow: 0 0 2px rgba(0,0,0,.5);
+}
 .history-thumbnail-popover {
     z-index: 2000;
     padding: 0;
@@ -3152,7 +3180,7 @@ div.dataTables_info {
 .channel-thumbnail {
     background-color: #868b8b;
     background-position: center;
-    background-size: cover;
+    background-size: contain;
     background-origin: content-box;
     background-repeat: no-repeat;
     height: 50px;
@@ -3797,9 +3825,8 @@ a:hover .overlay-refresh-image:hover {
 }
 
 .svg-icon {
-    padding: 10px;
+    background-size: calc(100% - 20px) calc(100% - 20px) !important;
     background-origin: content-box !important;
-    background-size: contain !important;
     background-repeat: no-repeat !important;
     background-position: center !important;
 }
@@ -3847,6 +3874,10 @@ a:hover .overlay-refresh-image:hover {
     background-color: #31afe1;
     background-image: url(../images/platforms/kodi.svg);
 }
+.platform-lg {
+    background-color: #a50034;
+    background-image: url(../images/platforms/lg.svg);
+}
 .platform-linux {
     background-color: #1793d0;
     background-image: url(../images/platforms/linux.svg);
@@ -3909,7 +3940,7 @@ a:hover .overlay-refresh-image:hover {
 }
 .platform-xbmc {
     background-color: #3b4872;
-    background-image: url(../images/platforms/xbmc.svg);
+    background-image: url(../images/platforms/kodi.svg);
 }
 .platform-xbox {
     background-color: #107c10;
@@ -3948,6 +3979,9 @@ a:hover .overlay-refresh-image:hover {
 .platform-kodi-rgba {
     background-color: rgba(49, 175, 225, 0.40);
 }
+.platform-lg-rgba {
+    background-color: rgba(165, 0, 52, 0.40);
+}
 .platform-linux-rgba {
     background-color: rgba(23, 147, 208, 0.40);
 }
@@ -4020,7 +4054,33 @@ a:hover .overlay-refresh-image:hover {
 .stats-most_concurrent {
     background-image: url(../images/icons/most-concurrent-streams.svg);
 }
-
+.rating-image {
+    width: 51px;
+    height: 20px;
+    margin-left: 10px;
+    display: inline-block;
+    background-origin: content-box !important;
+    background-size: contain !important;
+    background-repeat: no-repeat !important;
+    background-position: left !important;
+    text-align: right;
+}
+.rating-imdb {
+    width: 62px !important;
+    background-image: url(../images/rating/imdb.svg);
+}
+.rating-rottentomatos-ripe {
+    background-image: url(../images/rating/tomato-ripe.svg);
+}
+.rating-rottentomatos-rotten {
+    background-image: url(../images/rating/tomato-rotten.svg);
+}
+.rating-rottentomatos-upright {
+    background-image: url(../images/rating/popcorn-upright.svg);
+}
+.rating-rottentomatos-spilled {
+    background-image: url(../images/rating/popcorn-spilled.svg);
+}
 .transparent {
     background-color: transparent !important;
 }
@@ -4247,4 +4307,41 @@ a[data-tab-destination] {
 .help-block li {
     margin-top: 0;
     color: #737373;
-}
+}
+
+#browse-path-list > li > span > i.fa {
+    color: #999;
+}
+
+#tautulli-news .open .news-title,
+#tautulli-news .open .news-date,
+#tautulli-news .accordion li.open .link i.fa {
+    color: #eee;
+}
+.news-title,
+.news-date {
+    color: #999;
+    padding-left: 5px;
+}
+.news-subtitle {
+    display: block;
+    color: #aaa;
+    font-weight: bold;
+    margin-bottom: 10px;
+}
+.news-body {
+    display: block;
+    color: #aaa;
+}
+.news-body p:last-of-type {
+    margin-bottom: 0;
+}
+.news-body a {
+    display: inline !important;
+    background: none !important;
+    padding: 0 !important;
+    color: #fff;
+}
+.news-body a:hover {
+    color: #f9be03;
+}

data/interfaces/default/current_activity_instance.html

@@ -143,7 +143,7 @@ DOCUMENTATION :: END
                 <div id="platform-${sk}" class="dashboard-activity-info-platform${no_terminate} svg-icon platform-${data['platform_name']}" title="${data['platform']}"></div>
                 % if _session['user_group'] == 'admin' and plexpy.CONFIG.PMS_PLEXPASS and data['session_id']:
                 <div class="dashboard-activity-terminate-session" id="terminate-button-${sk}" data-key="${sk}" data-id="${data['session_id']}" data-toggle="tooltip" title="Terminate Stream">
-                    <i class="fa fa-times" style="padding-top: 8px;"></i>
+                    <i class="fa fa-times" style="padding-top: 10px;"></i>
                 </div>
                 % endif
             </div>
@@ -276,14 +276,17 @@ DOCUMENTATION :: END
                                 <div class="sub-heading">Subtitle</div>
                                 <div class="sub-value" id="subtitle_decision-${sk}">
                                     % if data['subtitles'] == 1:
+                                        <%
+                                            subtitle_codec = 'None' if data['stream_subtitle_codec'] and data['stream_subtitle_transient'] else data['subtitle_codec'].upper()
+                                        %>
                                         % if data['stream_subtitle_decision'] == 'transcode':
-                                        Transcode (${data['subtitle_codec'].upper()} <i class="fa fa-long-arrow-right"></i> ${data['stream_subtitle_codec'].upper()})
+                                        Transcode (${subtitle_codec} <i class="fa fa-long-arrow-right"></i> ${data['stream_subtitle_codec'].upper()})
                                         % elif data['stream_subtitle_decision'] == 'copy':
-                                        Direct Stream (${data['subtitle_codec'].upper()})
+                                        Direct Stream (${subtitle_codec})
                                         % elif data['stream_subtitle_decision'] == 'burn':
-                                        Burn (${data['subtitle_codec'].upper()})
+                                        Burn (${subtitle_codec})
                                         % else:
-                                        Direct Play (${data['subtitle_codec'].upper() if data['synced_version'] else data['stream_subtitle_codec'].upper()})
+                                        Direct Play (${subtitle_codec if data['synced_version'] else data['stream_subtitle_codec'].upper()})
                                         % endif
                                     % else:
                                     None
@@ -305,7 +308,11 @@ DOCUMENTATION :: END
                                     % endif
                                     <span id="location-${sk}">${data['location'].upper()}</span>:
                                     % if data['ip_address'] != 'N/A':
-                                    <span class="ip-container"><span class="ip-address">${data['ip_address']}</span></span>
+                                        % if len(data['ip_address']) > 20:
+                                        <span class="ip-container"><span class="ip-address" data-toggle="tooltip" title="${data['ip_address']}">${data['ip_address']}</span></span>
+                                        % else:
+                                        <span class="ip-container"><span class="ip-address">${data['ip_address']}</span></span>
+                                        % endif
                                         % if data['relayed']:
                                         <span data-toggle="tooltip" title="Plex Relay"><i class="fa fa-exclamation-circle"></i></span>
                                         % else:

data/interfaces/default/edit_library.html

@@ -40,13 +40,22 @@ DOCUMENTATION :: END
         <div class="modal-body" id="modal-text">
             <fieldset>
                 <div class="form-group">
-                    <label for="profile_url">Library Picture URL</label>
+                    <label for="profile_url">Library Thumbnail URL</label>
                     <div class="row">
                         <div class="col-md-8">
                             <input type="text" class="form-control" id="custom_thumb_url" name="custom_thumb_url" value="${data['library_thumb']}">
                         </div>
                     </div>
-                    <p class="help-block">Change the library's picture in Tautulli. To reset to default, leave this field empty and save.</p>
+                    <p class="help-block">Change the library's thumbnail in Tautulli. To reset to default, leave this field empty and save.</p>
+                </div>
+                <div class="form-group">
+                    <label for="profile_url">Library Background Art URL</label>
+                    <div class="row">
+                        <div class="col-md-8">
+                            <input type="text" class="form-control" id="custom_art_url" name="custom_art_url" value="${data['library_art']}">
+                        </div>
+                    </div>
+                    <p class="help-block">Change the library's background art in Tautulli. To reset to default, leave this field empty and save.</p>
                 </div>
                 <div class="checkbox">
                     <label>
@@ -80,6 +89,7 @@ DOCUMENTATION :: END
     // Save library options
     $("#save_library").on('click', function () {
         var custom_thumb = $("#custom_thumb_url").val();
+        var custom_art = $("#custom_art_url").val();
         var keep_history = 0;
         if ($("#keep_history").is(":checked")) {
             keep_history = 1;
@@ -90,6 +100,7 @@ DOCUMENTATION :: END
             data: {
                 section_id: '${data["section_id"]}',
                 custom_thumb: custom_thumb,
+                custom_art: custom_art,
                 keep_history: keep_history
             },
             cache: false,

data/interfaces/default/history.html

@@ -8,6 +8,13 @@
 
 <%def name="body()">
 <div class='container-fluid'>
+    % if config['database_is_importing']:
+    <div style="text-align: center; margin-top: 20px;">
+        <i class="fa fa-refresh fa-spin"></i>&nbsp; Tautulli is importing history from another database. This could take a few minutes depending on the size of your database.
+        <br />
+        You may leave this page and check back later.
+    </div>
+    % endif
     <div class='table-card-header'>
         <div class="header-bar">
             <span><i class="fa fa-history"></i> History</span>
@@ -185,19 +192,17 @@
                     $('#deleteCount').text(history_to_delete.length);
                     $('#confirm-modal-delete').modal();
                     $('#confirm-modal-delete').one('click', '#confirm-delete', function () {
-                        history_to_delete.forEach(function(row, idx) {
-                            $.ajax({
-                                url: 'delete_history_rows',
-                                type: 'POST',
-                                data: { row_id: row },
-                                async: true,
-                                success: function (data) {
-                                    var msg = "History deleted";
-                                    showMsg(msg, false, true, 2000);
-                                }
-                            });
+                        $.ajax({
+                            url: 'delete_history_rows',
+                            type: 'POST',
+                            data: { row_ids: history_to_delete.join(',') },
+                            async: true,
+                            success: function (data) {
+                                var msg = "History deleted";
+                                showMsg(msg, false, true, 2000);
+                                history_table.draw();
+                            }
                         });
-                        history_table.draw();
                     });
                 }
 

data/interfaces/default/home_stats.html

@@ -183,76 +183,6 @@ DOCUMENTATION :: END
 </div>
 % endif
 % endfor
-<script>
-    $('.dashboard-stats-instance .dashboard-stats-info-scroller').scrollbar();
-
-    function changeImages(elem) {
-        var stat_id = $(elem).data('stat_id');
-        var art = $(elem).data('art');
-        var thumb = $(elem).data('thumb');
-        var user_id = $(elem).data('user_id');
-        var user_thumb = $(elem).data('user_thumb');
-        var rating_key = $(elem).data('rating_key');
-        var guid = $(elem).data('guid');
-        var live = $(elem).data('live');
-        var [height, fallback_poster, fallback_art] = [450, 'poster', 'art'];
-        if ($.inArray(stat_id, ['top_music', 'popular_music']) > -1) {
-            [height, fallback_poster, fallback_art] = [300, 'cover', 'art'];
-        } else if (live) {
-            [height, fallback_poster, fallback_art] = [450, 'poster-live', 'art-live'];
-        }
-        var href = '#';
-
-        if (stat_id === 'most_concurrent') {
-            return
-        } else if (stat_id === 'top_users') {
-            $('#stats-thumb-' + stat_id).css('background-image', 'url(' + (user_thumb || 'images/gravatar-default.png') + ')');
-            if (user_id) {
-                href = page('user', user_id);
-            }
-            $('#stats-thumb-url-' + stat_id).attr('href', href).prop('title', $(elem).data('friendly_name'));
-        } else if (stat_id === 'top_platforms') {
-            $('#stats-thumb-' + stat_id).removeClass(function (index, className) {
-                return (className.match (/(^|\s)platform-\S+/g) || []).join(' ');
-            }).addClass('platform-' + $(elem).data('platform'));
-            $('#stats-background-' + stat_id).removeClass(function (index, className) {
-                return (className.match (/(^|\s)platform-\S+/g) || []).join(' ');
-            }).addClass('platform-' + $(elem).data('platform') + '-rgba');
-        } else {
-            if (rating_key) {
-                if (live) {
-                    href = page('info', rating_key, guid, true, live);
-                } else {
-                    href = page('info', rating_key);
-                }
-            }
-            $('#stats-thumb-url-' + stat_id).attr('href', href).prop('title', $(elem).data('title'));
-            $('#stats-background-' + stat_id).css('background-image', 'url(' + page('pms_image_proxy', art, rating_key, 500, 280, 40, '282828', 3, fallback_art) + ')');
-            $('#stats-thumb-' + stat_id).css('background-image', 'url(' + page('pms_image_proxy', thumb, rating_key, 300, height, null, null, null, fallback_poster) + ')');
-            $('#stats-thumb-' + stat_id + '-bg').css('background-image', 'url(' + page('pms_image_proxy', thumb, rating_key, 300, height, 60, '282828', 3, fallback_poster) + ')');
-        }
-    }
-
-    $('.dashboard-stats-info-item').mouseenter(function () {
-        changeImages(this);
-        if ($(this).data('stat_id') === 'last_watched') {
-            var friendly_name = $(this).data('friendly_name');
-            var last_watch = moment($(this).data('last_watch'), 'X').format(date_format);
-            $('#last-watched-header-info').html(friendly_name);
-        } else if ($(this).data('stat_id') === 'most_concurrent') {
-            var started = moment($(this).data('started'), 'X').format(date_format + ' ' + time_format);
-            $('#most-concurrent-header-info').html(started);
-        }
-    });
-    $('.dashboard-stats-instance').mouseleave(function () {
-        changeImages($(this).find('.dashboard-stats-info-item').first());
-        if ($(this).data('stat_id') === 'last_watched') {
-            $('#last-watched-header-info').text($(this).find('.dashboard-stats-info-item').first().data('friendly_name'));
-        } else if ($(this).data('stat_id') === 'most_concurrent') {
-            $('#most-concurrent-header-info').text('streams');
-        }
-    });
-</script>
 % else:
 <div class="text-muted">No stats to show for the selected period.</div><br>
 % endif

data/interfaces/default/images/favicon/browserconfig.xml

@@ -2,7 +2,7 @@
 <browserconfig>
     <msapplication>
         <tile>
-            <square150x150logo src="${http_root}images/favicon/mstile-150x150.png?v=2.0.5"/>
+            <square150x150logo src="mstile-150x150.png?v=2.0.5"/>
             <TileColor>#282a2d</TileColor>
         </tile>
     </msapplication>

data/interfaces/default/images/favicon/manifest.json

@@ -1,18 +1,23 @@
 {
-    "name": "Tautulli",
+    "name": "Tautulli: Monitor your Plex Media Server",
+    "short_name": "Tautulli",
+    "Description": "A Python based monitoring and tracking tool for Plex Media Server.",
+    "start_url": "../../",
+    "scope": "../../",
     "icons": [
         {
-            "src": "${http_root}images/favicon/android-chrome-192x192.png?v=2.0.5",
+            "src": "android-chrome-192x192.png?v=2.0.5",
             "sizes": "192x192",
             "type": "image/png"
         },
         {
-            "src": "${http_root}images/favicon/android-chrome-256x256.png?v=2.0.5",
+            "src": "android-chrome-256x256.png?v=2.0.5",
             "sizes": "256x256",
             "type": "image/png"
         }
     ],
     "theme_color": "#282a2d",
     "background_color": "#282a2d",
-    "display": "standalone"
+    "display": "standalone",
+    "orientation": "any"
 }

data/interfaces/default/images/platforms/lg.svg

@@ -0,0 +1,7 @@
+<!-- Generated by IcoMoon.io -->
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="64" height="64" viewBox="0 0 64 64">
+<title>lg</title>
+<path fill="#fff" d="M30.203 31.797c0 8.176-6.654 14.832-14.835 14.82-7.927-0.011-14.818-6.28-14.812-14.838 0.005-8.282 6.541-14.82 14.841-14.803 8.618 0.017 14.807 6.969 14.806 14.822zM26.577 32.388c-0.087 4.433-3.485 9.518-9.37 10.487-6.122 1.008-11.584-2.989-12.814-8.656-0.632-2.912-0.221-5.696 1.362-8.228 2.347-3.754 5.815-5.502 10.222-5.453 0-0.387 0-0.761 0-1.134-4.114-0.281-9.226 1.824-11.763 6.923-2.454 4.932-1.296 10.953 2.811 14.672 4.153 3.762 10.224 4.309 14.953 1.326 2.328-1.468 3.999-3.496 4.997-6.067 0.628-1.617 0.882-3.296 0.813-5.032-2.967 0-5.909 0-8.864 0 0 0.39 0 0.768 0 1.162 2.558-0 5.097-0 7.652-0zM15.991 37.112c0-0.129 0-0.221 0-0.313 0-3.731 0-7.463 0-11.194 0-0.060-0.004-0.119 0-0.179 0.009-0.118-0.038-0.166-0.16-0.163-0.278 0.006-0.556 0.012-0.833-0.002-0.178-0.008-0.237 0.042-0.237 0.23 0.005 4.194 0.005 8.389-0 12.583-0 0.198 0.065 0.239 0.249 0.237 1.224-0.007 2.448-0.004 3.672-0.004 0.072 0 0.143 0 0.244 0 0-0.343-0.008-0.665 0.003-0.987 0.006-0.166-0.050-0.214-0.214-0.212-0.82 0.007-1.641 0.003-2.461 0.003-0.078 0-0.155 0-0.263 0zM12.434 27.068c0.003-0.987-0.799-1.798-1.785-1.805s-1.799 0.796-1.805 1.782c-0.006 0.985 0.799 1.8 1.783 1.805 0.985 0.004 1.804-0.803 1.807-1.783z"></path>
+<path fill="#fff" d="M63.467 30.606c0 2.864 0 5.707 0 8.571-1.242 0-2.479 0-3.742 0 0-0.468 0-0.933 0-1.433-0.203 0.226-0.366 0.432-0.553 0.612-0.683 0.656-1.518 1-2.441 1.136-1.187 0.174-2.348 0.075-3.462-0.4-1.234-0.526-2.145-1.407-2.8-2.565-0.599-1.058-0.906-2.207-1.035-3.409-0.148-1.367-0.103-2.723 0.28-4.051 0.797-2.764 2.635-4.391 5.453-4.899 1.534-0.277 3.058-0.208 4.54 0.311 1.243 0.436 2.298 1.139 3.011 2.276 0.431 0.688 0.584 1.467 0.687 2.258 0.013 0.097 0.028 0.195 0.046 0.318-0.064 0.003-0.126 0.010-0.188 0.010-1.389 0.001-2.779-0.002-4.169 0.003-0.151 0.001-0.215-0.034-0.245-0.197-0.229-1.234-1.281-1.773-2.308-1.679-1.182 0.108-1.823 0.859-2.22 1.888-0.211 0.547-0.315 1.12-0.352 1.703-0.066 1.061-0.039 2.117 0.31 3.138 0.211 0.618 0.523 1.173 1.050 1.579 1.371 1.055 3.326 0.436 3.877-1.228 0.090-0.274 0.157-0.557 0.246-0.875-0.112 0-0.182 0-0.251 0-0.794 0-1.588-0.005-2.382 0.004-0.168 0.002-0.213-0.053-0.212-0.214 0.006-0.887 0.005-1.774 0.001-2.66-0.001-0.139 0.019-0.215 0.192-0.215 2.17 0.006 4.341 0.004 6.511 0.004 0.045 0 0.090 0.008 0.157 0.013z"></path>
+<path fill="#fff" d="M48.501 35.522c0 1.233 0 2.44 0 3.661-3.613 0-7.216 0-10.834 0 0-4.923 0-9.841 0-14.77 1.44 0 2.872 0 4.331 0 0 0.092 0 0.175 0 0.259 0 3.526 0 7.053 0 10.579 0 0.271 0 0.271 0.267 0.271 1.985 0 3.97 0 5.954 0 0.086 0 0.171 0 0.281 0z"></path>
+</svg>

data/interfaces/default/images/rating/imdb.svg

@@ -0,0 +1 @@
+<svg viewBox="0 0 1000 560" xmlns="http://www.w3.org/2000/svg" stroke-miterlimit="1.414" clip-rule="evenodd" fill-rule="evenodd" stroke-linejoin="round"><path d="M0 89.996C0 62.384 22.378 40 49.997 40h900.006C977.616 40 1000 62.388 1000 89.996v380.008c0 27.612-22.378 49.996-49.997 49.996H49.997C22.384 520 0 497.612 0 470.004V89.996z" fill="#e1be00"/><path d="M769.68 134.76v94.64c6.03-6.976 12.753-12.181 20.17-15.61 7.419-3.428 18.552-5.157 27.24-5.157 10.01 0 18.685 1.552 26.04 4.667 7.362 3.109 12.967 7.471 16.829 13.08 3.857 5.614 6.172 11.11 6.962 16.485.781 5.377 1.176 16.843 1.176 34.41v81.63c0 17.448-1.176 30.434-3.528 38.981-2.357 8.543-7.881 15.958-16.567 22.23-8.691 6.267-19 9.405-30.952 9.405-8.567 0-19.648-1.857-27.07-5.581-7.424-3.724-14.21-9.314-20.362-16.767l-4.709 18.538h-68.04v-290.95h72.809m-631.58 290.95h75.58v-290.95h-75.58v290.95m199.38-290.95c2.881 17.615 5.9 38.29 9.06 62.01l10.829 73.915 17.505-135.92h98.73v290.95h-65.99l-.239-196.38-26.433 196.38h-47.15l-27.862-192.11-.238 192.11h-66.2v-290.95h97.99m218.36 0c36.581 0 57.629 1.681 70.52 5.03 12.895 3.347 22.705 8.847 29.419 16.504 6.719 7.657 10.915 16.181 12.595 25.567 1.677 9.39 2.752 27.843 2.752 55.36v102.18c0 26.08-1.461 43.519-3.918 52.31-2.462 8.8-6.748 15.676-12.862 20.638-6.124 4.962-13.676 8.433-22.672 10.404-9 1.977-22.551 2.962-40.657 2.962h-91.57v-290.95h56.39m239.33 220.35c0 14.08-.7 22.977-2.096 26.677-1.4 3.704-7.485 5.566-12.1 5.566-4.5 0-7.5-1.786-9.02-5.371-1.519-3.581-2.272-11.757-2.272-24.538v-76.891c0-13.257.667-21.519 2-24.809 1.333-3.277 4.248-4.924 8.743-4.924 4.609 0 10.796 1.871 12.376 5.633 1.576 3.762 2.367 11.795 2.367 24.09v74.57m-203.37-167.99c2.986 1.728 4.901 4.457 5.734 8.157.833 3.709 1.257 12.138 1.257 25.29v112.8c0 19.371-1.257 31.23-3.767 35.595-2.509 4.371-9.2 6.548-20.06 6.548v-190.99c8.234 0 13.852.866 16.838 2.6"/></svg>

data/interfaces/default/images/rating/popcorn-spilled.svg

@@ -0,0 +1,8 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="560" height="560" viewBox="0 0 560 560">
+  <g fill="none" transform="translate(33 140)">
+    <path fill="#FFF" d="M43.8020066,267.3152 L281.745403,290.797105 C286.539148,305.7344 292.894623,320.31421 302.004138,331.0528 L71.7380852,300.927619 C61.4905377,294.175695 50.8770689,281.050362 43.8020066,267.3152 Z M266.684852,192.017143 C267.285384,212.923048 270.116459,239.981562 275.03101,263.034133 L33.8766098,243.950705 C26.3585902,221.21181 24.03,207.991848 22.1723803,189.066133 L266.684852,192.017143 Z M275.03101,89.3083429 C270.116459,112.360914 267.285384,139.419429 266.684852,160.325333 L22.1723803,163.276343 C24.03,144.350629 26.3585902,131.130667 33.8766098,108.391771 L275.03101,89.3083429 Z M302.004138,21.2896762 C292.894623,32.030019 286.539148,46.6080762 281.745403,61.5471238 L43.8020066,85.0272762 C50.8770689,71.2921143 61.4905377,58.1685333 71.7380852,51.4148571 L302.004138,21.2896762 Z"/>
+    <path fill="#00641E" d="M303.565869,264.667352 C306.720846,256.846476 317.903331,252.081752 325.93259,252.63901 C334.515108,253.234819 343.631626,262.264838 345.224872,271.145905 C345.520761,270.823467 345.830656,270.518552 346.145803,270.218895 C348.901593,267.583314 352.35421,265.834438 356.132479,265.352533 C355.554708,262.790552 355.416393,260.048076 355.812079,257.233752 C357.207482,247.3328 365.145698,239.907962 374.234203,239.981562 C380.099449,240.028876 385.245108,243.032457 388.597928,247.646476 C388.897318,247.271467 389.228223,246.928 389.550374,246.577524 C393.384669,226.586362 395.814807,203.999924 396.368066,180.030857 C398.267705,97.6111238 377.375174,30.2776381 349.70522,29.6397714 C322.033515,29.0001524 298.061292,95.2962286 296.161652,177.715962 C296.161652,177.715962 294.696216,207.778057 303.565869,264.667352"/>
+    <path fill="#FFD700" d="M490.910577,354.797562 C492.545843,352.0656 493.45977,348.7904 493.396741,345.310171 C493.927239,334.065143 486.214879,323.871543 475.484105,325.000076 C475.794,323.713829 475.997095,322.376762 476.075882,320.997638 C476.718433,309.733333 468.957049,300.025143 458.739266,299.315429 C458.515161,299.30141 458.294557,299.2944 458.072203,299.28739 C459.134951,296.492343 459.661948,293.371352 459.47461,290.06461 C458.945862,280.692876 452.488839,272.796648 444.088407,271.242286 C441.054236,270.681524 438.111108,270.960152 435.416597,271.894171 C432.870905,265.617143 427.525652,260.961067 421.023108,259.977981 C420.508367,249.786133 413.153174,241.397486 403.68299,240.740343 C397.728452,240.326781 392.257141,243.064 388.597928,247.646476 C385.245108,243.032457 380.099449,240.030629 374.234203,239.983314 C365.145698,239.907962 357.207482,247.3328 355.812079,257.235505 C355.416393,260.048076 355.554708,262.790552 356.132479,265.354286 C352.35421,265.834438 348.901593,267.585067 346.145803,270.218895 C345.830656,270.518552 345.520761,270.823467 345.224872,271.145905 C343.631626,262.264838 334.515108,253.236571 325.93259,252.63901 C317.903331,252.081752 306.575528,256.963886 303.565869,264.667352 C304.885987,278.101105 313.275915,314.72061 343.621121,347.570743 L343.890748,347.590019 C346.816367,350.239619 350.636656,351.699352 354.709062,351.33661 C357.233744,351.110552 359.558833,350.206324 361.56177,348.811429 L362.050249,348.844724 C364.720249,350.700495 367.929502,351.671314 371.320839,351.369905 C372.616446,351.254248 373.852525,350.942324 375.027325,350.497219 C377.933685,356.51139 384.41522,360.398171 391.649607,359.760305 C397.25223,359.266133 402.014459,356.164419 404.81402,351.799238 L405.720944,351.862324 C408.517003,354.636343 412.233993,356.252038 416.24337,356.131124 C419.557672,361.149943 425.645272,364.237638 432.363167,363.647086 C434.893102,363.424533 437.254957,362.692038 439.361193,361.582781 C442.875089,365.90941 448.642289,368.481905 454.969751,367.924648 C461.22543,367.376152 466.509403,363.904686 469.384249,359.104914 C472.208321,361.374248 475.746728,362.592152 479.503987,362.257448 C483.149193,361.931505 486.38821,360.208914 488.821849,357.603124 L489.228039,357.631162 C489.781298,356.828571 490.25402,356.006705 490.693475,355.177829 C490.70398,355.162057 490.712734,355.144533 490.721489,355.128762 C490.781016,355.018362 490.854551,354.909714 490.910577,354.797562"/>
+    <path fill="#04A53C" d="M281.745403,61.5471238 L43.8020066,85.0272762 C50.8770689,71.2921143 61.4905377,58.1685333 71.7380852,51.4148571 L302.004138,21.2896762 C292.894623,32.030019 286.539148,46.6080762 281.745403,61.5471238 Z M302.004138,331.0528 L71.7380852,300.927619 C61.4905377,294.175695 50.8770689,281.050362 43.8020066,267.316952 L281.745403,290.797105 C286.539148,305.7344 292.894623,320.31421 302.004138,331.0528 Z M33.8766098,243.950705 C26.3585902,221.21181 24.03,207.9936 22.1723803,189.066133 L266.684852,192.017143 C267.285384,212.923048 270.116459,239.981562 275.03101,263.034133 L33.8766098,243.950705 Z M33.8766098,108.391771 L275.03101,89.3083429 C270.116459,112.360914 267.285384,139.419429 266.684852,160.327086 L22.1723803,163.276343 C24.03,144.350629 26.3585902,131.130667 33.8766098,108.391771 Z M378.597246,25.7126857 C363.342354,7.93478095 352.390977,-0.411809524 343.010085,0.672914286 C341.261016,0.895466667 76.1168852,37.8952381 76.1168852,37.8952381 C34.0429377,42.1780571 0.416695082,103.32739 0,176.172114 C0.416695082,249.015086 34.0429377,310.164419 76.1168852,314.44899 C76.1168852,314.44899 341.758249,351.583695 343.010085,351.669562 C345.228374,351.655543 347.418649,351.357638 349.57741,350.803886 C347.476426,350.178286 345.538269,349.083048 343.890748,347.590019 L343.621121,347.570743 C313.275915,314.722362 304.885987,278.101105 303.565869,264.667352 C303.56937,264.656838 303.576374,264.648076 303.579875,264.637562 C303.576374,264.648076 303.56937,264.656838 303.565869,264.667352 C294.696216,207.778057 296.161652,177.715962 296.161652,177.715962 C298.061292,95.2962286 322.033515,29.0001524 349.70522,29.638019 C377.375174,30.2776381 398.267705,97.6111238 396.368066,180.030857 C395.814807,203.999924 393.384669,226.586362 389.550374,246.577524 C393.489718,242.226362 398.773692,240.393371 403.68299,240.740343 C404.586413,240.805181 405.465325,240.957638 406.326728,241.155657 C423.131095,149.125867 405.514348,59.262019 378.597246,25.7126857 Z"/>
+  </g>
+</svg>

data/interfaces/default/images/rating/popcorn-upright.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="560" height="560"><g fill="none"><path fill="#FFF" d="M370.57 474.214l23.466-237.956c14.93-4.796 29.498-11.15 40.23-20.262L404.16 446.278c-6.748 10.248-19.863 20.86-33.59 27.936zm-78.197 21.631l2.947-244.528c20.894-.599 47.933-3.43 70.97-8.346l-19.07 241.17c-22.724 7.518-35.934 9.848-54.847 11.704zm-99.694-252.874c23.038 4.916 50.077 7.747 70.971 8.346l2.948 244.528c-18.914-1.856-32.123-4.186-54.847-11.705l-19.072-241.17zm-67.974-26.975c10.732 9.112 25.3 15.466 40.23 20.262l23.464 237.956c-13.726-7.075-26.84-17.688-33.59-27.936l-30.104-230.282z"/><path fill="gold" d="M118.905 157.445c1.357 28.827 72.771 51.677 160.578 51.176 76.687-.438 140.659-18.546 156.329-42.336a22.976 22.976 0 00-14.058-7.426c.06-.7.098-1.406.095-2.122-.065-11.4-8.429-20.788-19.327-22.54.287-1.474.438-2.999.43-4.559-.072-12.696-10.426-22.928-23.124-22.856-.287.001-.568.036-.853.049a22.911 22.911 0 001.254-7.56c-.074-12.697-10.425-22.93-23.123-22.858a22.914 22.914 0 00-8.247 1.6c-3.632-6.835-10.606-11.6-18.737-12.149-1.416-11.4-11.157-20.195-22.93-20.129-7.41.042-13.963 3.6-18.136 9.065-4.233-4.605-10.3-7.494-17.047-7.456-12.698.072-22.932 10.424-22.86 23.118a22.983 22.983 0 001.115 6.946 22.918 22.918 0 00-13.07 7.459c-2.644-9.847-11.637-17.084-22.314-17.024-9.975.057-18.406 6.47-21.537 15.366-8.474 3.426-14.439 11.738-14.383 21.433.012 2.154.342 4.227.907 6.202a22.876 22.876 0 00-9.328-1.932c-10.012.058-18.47 6.516-21.574 15.465a22.83 22.83 0 00-9.788-2.149c-12.698.072-22.934 10.422-22.86 23.118a22.833 22.833 0 003.159 11.463c-.202.203-.379.426-.571.636"/><path fill="#FA320A" d="M404.161 446.278c-6.749 10.248-19.864 20.86-33.59 27.936l23.465-237.956c14.93-4.796 29.498-11.15 40.23-20.262L404.16 446.278zM347.22 484.14c-22.723 7.519-35.934 9.85-54.847 11.705l2.947-244.528c20.894-.599 47.933-3.43 70.973-8.346L347.22 484.14zm-135.47 0l-19.07-241.17c23.037 4.917 50.076 7.748 70.97 8.347l2.948 244.528c-18.914-1.856-32.123-4.186-54.847-11.705zm-56.94-37.862l-30.105-230.282c10.732 9.112 25.3 15.466 40.23 20.262l23.464 237.956c-13.726-7.075-26.84-17.688-33.588-27.936zm247.668-321.143c.298 1.453.465 2.955.473 4.498a23.018 23.018 0 01-.43 4.56c10.9 1.749 19.263 11.137 19.328 22.54a23.59 23.59 0 01-.095 2.12 22.976 22.976 0 0114.058 7.425c-15.669 23.792-79.642 41.9-156.327 42.34-87.807.502-159.221-22.346-160.58-51.175.192-.208.37-.433.57-.634-1.355-2.311-2.29-4.887-2.773-7.62-8.408 7.979-13.495 14.412-12.6 23.78.085 1.251 37.196 266.911 37.196 266.911 4.282 42.075 65.391 75.703 138.187 76.12 72.796-.417 133.907-34.045 138.187-76.12 0 0 37.11-265.66 37.197-266.912 1.777-18.736-20.15-35.745-52.39-47.833z"/></g></svg>

data/interfaces/default/images/rating/tomato-ripe.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="560" height="560"><g fill="none"><path fill="#FA320A" d="M478.29 296.976c-3.99-63.966-36.52-111.823-85.468-138.579.278 1.56-1.109 3.508-2.688 2.818-32.016-14.006-86.328 31.32-124.282 7.584.285 8.519-1.378 50.072-59.914 52.483-1.382.056-2.142-1.355-1.268-2.354 7.828-8.929 15.732-31.535 4.367-43.586-24.338 21.81-38.472 30.017-85.138 19.186-29.878 31.241-46.809 74-43.485 127.265 6.78 108.735 108.63 170.89 211.193 164.49 102.556-6.395 193.466-80.572 186.683-189.307"/><path fill="#00912D" d="M291.375 132.293c21.075-5.023 81.693-.49 101.114 25.274 1.166 1.545-.475 4.468-2.355 3.648-32.016-14.006-86.328 31.32-124.282 7.584.285 8.519-1.378 50.072-59.914 52.483-1.382.056-2.142-1.355-1.268-2.354 7.828-8.929 15.73-31.535 4.367-43.586-26.512 23.758-40.884 31.392-98.426 15.838-1.883-.508-1.241-3.535.762-4.298 10.876-4.157 35.515-22.361 58.824-30.385 4.438-1.526 8.862-2.71 13.18-3.4-25.665-2.293-37.235-5.862-53.559-3.4-1.789.27-3.004-1.813-1.895-3.241 21.995-28.332 62.513-36.888 87.512-21.837-15.41-19.094-27.48-34.321-27.48-34.321l28.601-16.246s11.817 26.4 20.414 45.614c21.275-31.435 60.86-34.336 77.585-12.033.992 1.326-.045 3.21-1.702 3.171-13.612-.331-21.107 12.05-21.675 21.466l.197.023"/></g></svg>

data/interfaces/default/images/rating/tomato-rotten.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="560" height="560"><path fill="#0AC855" d="M445.185 444.684c-79.369 4.167-95.587-86.652-126.726-86.006-13.268.279-23.726 14.151-19.133 30.32 2.525 8.888 9.53 21.923 13.944 30.011 15.57 28.544-7.447 60.845-34.383 63.577-44.76 4.54-63.433-21.426-62.278-48.007 1.3-29.84 26.6-60.331.65-73.305-27.194-13.597-49.301 39.572-75.325 51.439-23.553 10.741-56.248 2.413-67.872-23.741-8.164-18.379-6.68-53.768 29.67-67.27 22.706-8.433 73.305 11.029 75.9-13.623 2.992-28.416-53.155-30.812-70.06-37.626-29.912-12.055-47.567-37.85-33.734-65.522 10.378-20.757 40.915-29.203 64.223-20.11 27.922 10.892 32.404 39.853 46.71 51.897 12.324 10.38 29.19 11.68 40.22 4.543 8.135-5.265 10.843-16.828 7.774-27.39-4.07-14.023-14.875-22.773-25.415-31.346-18.758-15.249-45.24-28.36-29.222-69.983 13.13-34.11 51.642-35.34 51.642-35.34 15.3-1.72 29.002 2.9 40.167 12.875 14.927 13.335 17.834 31.16 15.336 50.176-2.283 17.358-8.426 32.56-11.63 49.759-3.717 19.966 6.954 40.086 27.249 40.869 26.694 1.031 34.698-19.486 37.964-32.492 4.782-19.028 11.058-36.694 28.718-47.82 25.346-15.97 60.552-12.47 76.886 18.222 12.92 24.284 8.772 57.715-11.047 75.97-8.892 8.188-19.584 11.075-31.148 11.156-16.585.117-33.162-.29-48.556 7.471-10.48 5.281-15.047 13.888-15.045 25.423 0 11.242 5.853 18.585 15.336 23.363 17.86 9.003 37.577 10.843 56.871 14.222 27.98 4.9 52.581 14.755 68.375 40.72.142.228.28.458.415.69 18.139 30.741-.831 75.005-36.476 76.878"/></svg>

data/interfaces/default/index.html

@@ -493,14 +493,15 @@
 
                         var subtitle_decision = 'None';
                         if (['movie', 'episode', 'clip'].indexOf(s.media_type) > -1 && s.subtitles === 1) {
+                            var subtitle_codec = (s.stream_subtitle_codec && s.stream_subtitle_transient) ? 'None' : s.subtitle_codec.toUpperCase();
                             if (s.stream_subtitle_decision === 'transcode') {
-                                subtitle_decision = 'Transcode (' + s.subtitle_codec.toUpperCase() + ' <i class="fa fa-long-arrow-right"></i> ' + s.stream_subtitle_codec.toUpperCase() + ')';
+                                subtitle_decision = 'Transcode (' + subtitle_codec + ' <i class="fa fa-long-arrow-right"></i> ' + s.stream_subtitle_codec.toUpperCase() + ')';
                             } else if (s.stream_subtitle_decision === 'copy') {
-                                subtitle_decision = 'Direct Stream (' + s.subtitle_codec.toUpperCase() + ')';
+                                subtitle_decision = 'Direct Stream (' + subtitle_codec + ')';
                             } else if (s.stream_subtitle_decision === 'burn') {
-                                subtitle_decision = 'Burn (' + s.subtitle_codec.toUpperCase() + ')';
+                                subtitle_decision = 'Burn (' + subtitle_codec + ')';
                             } else {
-                                subtitle_decision = 'Direct Play (' + ((s.synced_version === '1') ? s.subtitle_codec.toUpperCase() : s.stream_subtitle_codec.toUpperCase()) + ')';
+                                subtitle_decision = 'Direct Play (' + ((s.synced_version === '1') ? subtitle_codec : s.stream_subtitle_codec.toUpperCase()) + ')';
                             }
                         }
                         $('#subtitle_decision-' + key).html(subtitle_decision);
@@ -726,6 +727,88 @@
     % endif
 </script>
 % endif
+% if 'watch_stats' in config['home_sections'] or 'library_stats' in config['home_sections']:
+<script>
+    function statsCardCallback() {
+        $('.dashboard-stats-instance .dashboard-stats-info-scroller').scrollbar();
+
+        function changeImages(elem) {
+            var stat_id = $(elem).data('stat_id');
+            var art = $(elem).data('art');
+            var thumb = $(elem).data('thumb');
+            var user_id = $(elem).data('user_id');
+            var user_thumb = $(elem).data('user_thumb');
+            var rating_key = $(elem).data('rating_key');
+            var guid = $(elem).data('guid');
+            var live = $(elem).data('live');
+            var [height, fallback_poster, fallback_art] = [450, 'poster', 'art'];
+            if ($.inArray(stat_id, ['top_music', 'popular_music']) > -1) {
+                [height, fallback_poster, fallback_art] = [300, 'cover', 'art'];
+            } else if (live) {
+                [height, fallback_poster, fallback_art] = [450, 'poster-live', 'art-live'];
+            }
+            var href = '#';
+
+            if (stat_id === 'most_concurrent') {
+                return
+            } else if (stat_id === 'top_users') {
+                $('#stats-thumb-' + stat_id).css('background-image', 'url(' + (user_thumb || 'images/gravatar-default.png') + ')');
+                if (user_id) {
+                    href = page('user', user_id);
+                }
+                $('#stats-thumb-url-' + stat_id).attr('href', href).prop('title', $(elem).data('friendly_name'));
+            } else if (stat_id === 'top_platforms') {
+                $('#stats-thumb-' + stat_id).removeClass(function (index, className) {
+                    return (className.match (/(^|\s)platform-\S+/g) || []).join(' ');
+                }).addClass('platform-' + $(elem).data('platform'));
+                $('#stats-background-' + stat_id).removeClass(function (index, className) {
+                    return (className.match (/(^|\s)platform-\S+/g) || []).join(' ');
+                }).addClass('platform-' + $(elem).data('platform') + '-rgba');
+            } else {
+                if (rating_key) {
+                    if (live) {
+                        href = page('info', rating_key, guid, true, live);
+                    } else {
+                        href = page('info', rating_key);
+                    }
+                }
+                $('#stats-thumb-url-' + stat_id).attr('href', href).prop('title', $(elem).data('title'));
+                $('#stats-background-' + stat_id).css('background-image', 'url(' + page('pms_image_proxy', art, rating_key, 500, 280, 40, '282828', 3, fallback_art) + ')');
+                $('#stats-thumb-' + stat_id).css('background-image', 'url(' + page('pms_image_proxy', thumb, rating_key, 300, height, null, null, null, fallback_poster) + ')');
+                $('#stats-thumb-' + stat_id + '-bg').css('background-image', 'url(' + page('pms_image_proxy', thumb, rating_key, 300, height, 60, '282828', 3, fallback_poster) + ')');
+                $('#library-stats-background-' + stat_id).css('background-image', 'url(' + page('pms_image_proxy', art, rating_key, 500, 280, 40, '282828', 3, fallback_art) + ')');
+                if (thumb.startsWith('http')) {
+                    $('#library-stats-thumb-' + stat_id).css('background-image', 'url(' + page('pms_image_proxy', thumb, rating_key, 300, 300, null, null, null, 'cover') + ')')
+                        .removeClass('svg-icon library-' + stat_id);
+                } else {
+                    $('#library-stats-thumb-' + stat_id).css('background-image', '')
+                        .addClass('svg-icon library-' + stat_id);
+                }
+            }
+        }
+
+        $('.dashboard-stats-info-item').mouseenter(function () {
+            changeImages(this);
+            if ($(this).data('stat_id') === 'last_watched') {
+                var friendly_name = $(this).data('friendly_name');
+                var last_watch = moment($(this).data('last_watch'), 'X').format(date_format);
+                $('#last-watched-header-info').html(friendly_name);
+            } else if ($(this).data('stat_id') === 'most_concurrent') {
+                var started = moment($(this).data('started'), 'X').format(date_format + ' ' + time_format);
+                $('#most-concurrent-header-info').html(started);
+            }
+        });
+        $('.dashboard-stats-instance').mouseleave(function () {
+            changeImages($(this).find('.dashboard-stats-info-item').first());
+            if ($(this).data('stat_id') === 'last_watched') {
+                $('#last-watched-header-info').text($(this).find('.dashboard-stats-info-item').first().data('friendly_name'));
+            } else if ($(this).data('stat_id') === 'most_concurrent') {
+                $('#most-concurrent-header-info').text('streams');
+            }
+        });
+    }
+</script>
+% endif
 % if 'watch_stats' in config['home_sections']:
 <script>
     function getHomeStats(time_range, stats_type) {
@@ -744,6 +827,7 @@
                 $("#home-stats").html(xhr.responseText);
                 $('#ajaxMsg').fadeOut();
                 lockScroll('#home-stats .dashboard-stats-info-scroller');
+                statsCardCallback();
             }
         });
     }
@@ -783,6 +867,7 @@
             data: { },
             complete: function (xhr, status) {
                 $("#library-stats").html(xhr.responseText);
+                statsCardCallback();
             }
         });
     }

data/interfaces/default/info.html

@@ -41,18 +41,18 @@ DOCUMENTATION :: END
 
     from plexpy import notifiers
     from plexpy.common import MEDIA_TYPE_HEADERS, MEDIA_FLAGS_AUDIO, MEDIA_FLAGS_VIDEO
-    from plexpy.helpers import page
+    from plexpy.helpers import page, get_percent
 
     # Get audio codec file
     def af(codec):
-        for pattern, file_type in MEDIA_FLAGS_AUDIO.iteritems():
+        for pattern, file_type in MEDIA_FLAGS_AUDIO.items():
             if re.match(pattern, codec):
                 return file_type
         return codec
 
     # Get video codec file
     def vf(codec):
-        for pattern, file_type in MEDIA_FLAGS_VIDEO.iteritems():
+        for pattern, file_type in MEDIA_FLAGS_VIDEO.items():
             if re.match(pattern, codec):
                 return file_type
         return codec
@@ -269,16 +269,28 @@ DOCUMENTATION :: END
                     <div class="summary-content">
                         <div class="summary-content-details-wrapper">
                             % if data['rating']:
-                            <div class="star-rating hidden-xs hidden-sm" title="${data['rating']}">
-                                % for i in range(0,5):
-                                % if round(float(data['rating']) / 2) > i:
-                                <i class="star-icon fa fa-star"></i>
-                                % else:
-                                <i class="star-icon-o fa fa-star-o"></i>
-                                % endif
-                                % endfor
+                            % if data['rating_image']:
+                            % if data['rating_image'].startswith('imdb://'):
+                            <div class="critic-rating hidden-xs hidden-sm" title="${data['rating']}">
+                                <span class="rating-image rating-imdb"><strong>${data['rating']}</strong></span>
                             </div>
                             % endif
+                            % if data['audience_rating_image'].startswith('rottentomatoes://'):
+                            <div class="critic-rating hidden-xs hidden-sm" title="${data['audience_rating']}">
+                                <span class="rating-image rating-rottentomatos-${data['audience_rating_image'].rsplit('.')[-1]}"><strong>${get_percent(data['audience_rating'], 10)}%</strong></span>
+                            </div>
+                            % endif
+                            % if data['rating_image'].startswith('rottentomatoes://'):
+                            <div class="critic-rating hidden-xs hidden-sm" title="${data['rating']}">
+                                <span class="rating-image rating-rottentomatos-${data['rating_image'].rsplit('.')[-1]}"><strong>${get_percent(data['rating'], 10)}%</strong></span>
+                            </div>
+                            % endif
+                            % else:
+                            <div class="critic-rating hidden-xs hidden-sm" title="${data['rating']}">
+                                <i class="star-icon fa fa-star"></i> <strong>${get_percent(data['rating'], 10)}%</strong>
+                            </div>
+                            % endif
+                            % endif
                             <div class="summary-content-details-tag">
                                 % if data['directors']:
                                 Directed by <strong> ${data['directors'][0]}</strong>
@@ -709,19 +721,17 @@ DOCUMENTATION :: END
                     $('#deleteCount').text(history_to_delete.length);
                     $('#confirm-modal-delete').modal();
                     $('#confirm-modal-delete').one('click', '#confirm-delete', function () {
-                        history_to_delete.forEach(function (row, idx) {
-                            $.ajax({
-                                url: 'delete_history_rows',
-                                type: 'POST',
-                                data: { row_id: row },
-                                async: true,
-                                success: function (data) {
-                                    var msg = "History deleted";
-                                    showMsg(msg, false, true, 2000);
-                                }
-                            });
+                        $.ajax({
+                            url: 'delete_history_rows',
+                            type: 'POST',
+                            data: { row_ids: history_to_delete.join(',') },
+                            async: true,
+                            success: function (data) {
+                                var msg = "History deleted";
+                                showMsg(msg, false, true, 2000);
+                                history_table.draw();
+                            }
                         });
-                        history_table.draw();
                     });
                 }
 
@@ -848,10 +858,10 @@ DOCUMENTATION :: END
 % if data.get('tvmaze_id') or data.get('themoviedb_id') or data.get('musicbrainz_id'):
 <script>
     $('#delete-lookup-info').on('click', function () {
-        var msg = 'Are you sure you want to delete the 3rd party API lookup for <strong>' + $(this).data('title') + '</strong>?<br><br>' +
-                  'The info will be looked up again the next time a notification is sent.';
+        var msg = 'Are you sure you want to delete all the metadata lookup info for <strong>' + $(this).data('title') + '</strong>?' +
+                  '<br /><br />Tautulli will lookup the metadata info again the next time a notification is sent.';
         var url = 'delete_lookup_info';
-        var data = { rating_key: $(this).data('id'), title: $(this).data('title') };
+        var data = { rating_key: $(this).data('id') };
         var callback = function () {
             $('#delete-lookup-info').closest('.btn-group').remove();
         };

data/interfaces/default/ip_address_modal.html

@@ -24,7 +24,6 @@
             <div id="ip_error" class="col-sm-12 text-muted"></div>
             <div class="col-sm-6">
                 <ul class="list-unstyled">
-                    <li>Continent: <strong><span id="continent"></span></strong></li>
                     <li>Country: <strong><span id="country"></span></strong></li>
                     <li>Region: <strong><span id="region"></span></strong></li>
                     <li>City: <strong><span id="city"></span></strong></li>
@@ -36,7 +35,6 @@
                     <li>Timezone: <strong><span id="timezone"></span></strong></li>
                     <li>Latitude: <strong><span id="latitude"></span></strong></li>
                     <li>Longitude: <strong><span id="longitude"></span></strong></li>
-                    <li>Accuracy Radius: <strong><span id="accuracy"></span></strong></li>
                 </ul>
             </div>
             <div class="col-sm-12">
@@ -61,8 +59,6 @@
             </div>
         </div>
         <div class="modal-footer">
-            <% from plexpy.helpers import anon_url %>
-            <span class="text-muted">GeoLite2 data created by <a href="${anon_url('http://www.maxmind.com')}" target="_blank">MaxMind</a>.</span>
         </div>
     </div>
 </div>
@@ -82,11 +78,11 @@
             error: function () {
                 $('#ip_error').html('<i class="fa fa-exclamation-circle"></i> Internal request failed.').show();
             },
-            success: function (data) {
-                if ('error' in data) {
-                    $('#ip_error').html('<i class="fa fa-exclamation-circle"></i> ' + data.error).show();
+            success: function (result) {
+                if (result.result === 'error') {
+                    $('#ip_error').html('<i class="fa fa-exclamation-circle"></i> ' + result.message).show();
                 } else {
-                    $('#continent').html(data.continent);
+                    var data = result.data;
                     $('#country').html(data.country);
                     $('#region').html(data.region);
                     $('#city').html(data.city);
@@ -94,7 +90,6 @@
                     $('#timezone').html(data.timezone);
                     $('#latitude').html(data.latitude);
                     $('#longitude').html(data.longitude);
-                    $('#accuracy').html(data.accuracy + ' km');
                 }
             }
         });

data/interfaces/default/js/ajaxNotifications.js

@@ -36,7 +36,3 @@ function check_notifications() {
         check_notifications();
     }, 5000);
 }
-
-$(document).ready(function () {
-    check_notifications();
-});

data/interfaces/default/js/script.js

@@ -237,6 +237,27 @@ function doAjaxCall(url, elem, reload, form, showMsg, callback) {
     });
 }
 
+getBrowsePath = function (key, path, filter_ext) {
+    var deferred = $.Deferred();
+
+    $.ajax({
+        url: 'browse_path',
+        type: 'GET',
+        data: {
+            key: key,
+            path: path,
+            filter_ext: filter_ext
+        },
+        success: function(data) {
+            deferred.resolve(data);
+        },
+        error: function() {
+            deferred.reject();
+        }
+    });
+    return deferred;
+};
+
 function doSimpleAjaxCall(url) {
     $.ajax(url);
 }
@@ -258,33 +279,31 @@ $.cachedScript = function (url) {
 function isPrivateIP(ip_address) {
     var defer = $.Deferred();
 
-    $.cachedScript('js/ipaddr.min.js').done(function () {
-        if (ipaddr.isValid(ip_address)) {
-            var addr = ipaddr.process(ip_address);
+    if (ipaddr.isValid(ip_address)) {
+        var addr = ipaddr.process(ip_address);
 
-            var rangeList = [];
-            if (addr.kind() === 'ipv4') {
-                rangeList = [
-                    ipaddr.parseCIDR('127.0.0.0/8'),
-                    ipaddr.parseCIDR('10.0.0.0/8'),
-                    ipaddr.parseCIDR('172.16.0.0/12'),
-                    ipaddr.parseCIDR('192.168.0.0/16')
-                ];
-            } else {
-                rangeList = [
-                    ipaddr.parseCIDR('fd00::/8')
-                ];
-            }
-
-            if (ipaddr.subnetMatch(addr, rangeList, -1) >= 0) {
-                defer.resolve();
-            } else {
-                defer.reject();
-            }
+        var rangeList = [];
+        if (addr.kind() === 'ipv4') {
+            rangeList = [
+                ipaddr.parseCIDR('127.0.0.0/8'),
+                ipaddr.parseCIDR('10.0.0.0/8'),
+                ipaddr.parseCIDR('172.16.0.0/12'),
+                ipaddr.parseCIDR('192.168.0.0/16')
+            ];
         } else {
-            defer.resolve('n/a');
+            rangeList = [
+                ipaddr.parseCIDR('fd00::/8')
+            ];
         }
-    });
+
+        if (ipaddr.subnetMatch(addr, rangeList, -1) >= 0) {
+            defer.resolve();
+        } else {
+            defer.reject();
+        }
+    } else {
+        defer.resolve('n/a');
+    }
 
     return defer.promise();
 }
@@ -463,8 +482,9 @@ $('*').on('click', '.refresh_pms_image', function (e) {
 });
 
 // Taken from http://stackoverflow.com/questions/10420352/converting-file-size-in-bytes-to-human-readable#answer-14919494
-function humanFileSize(bytes, si) {
-    var thresh = si ? 1000 : 1024;
+function humanFileSize(bytes, si = true) {
+    //var thresh = si ? 1000 : 1024;
+    var thresh = 1024;  // Always divide by 2^10 but display SI units
     if (Math.abs(bytes) < thresh) {
         return bytes + ' B';
     }

data/interfaces/default/js/tables/history_table.js

@@ -36,10 +36,10 @@ history_table_options = {
             "targets": [0],
             "data": null,
             "createdCell": function (td, cellData, rowData, row, col) {
-                if (rowData['id'] === null) {
+                if (rowData['row_id'] === null) {
                     $(td).html('');
                 } else {
-                    $(td).html('<button class="btn btn-xs btn-warning" data-id="' + rowData['id'] + '"><i class="fa fa-trash-o fa-fw"></i> Delete</button>');
+                    $(td).html('<button class="btn btn-xs btn-warning" data-id="' + rowData['row_id'] + '"><i class="fa fa-trash-o fa-fw"></i> Delete</button>');
                 }
             },
             "width": "5%",
@@ -317,19 +317,19 @@ history_table_options = {
     "rowCallback": function (row, rowData, rowIndex) {
         if (rowData['group_count'] == 1) {
             // if no grouped rows simply toggle the delete button
-            if ($.inArray(rowData['id'], history_to_delete) !== -1) {
-                $(row).find('button[data-id="' + rowData['id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
+            if ($.inArray(rowData['row_id'], history_to_delete) !== -1) {
+                $(row).find('button[data-id="' + rowData['row_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
             }
-        } else if (rowData['id'] !== null) {
+        } else if (rowData['row_id'] !== null) {
             // if grouped rows
             // toggle the parent button to danger
-            $(row).find('button[data-id="' + rowData['id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
+            $(row).find('button[data-id="' + rowData['row_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
             // check if any child rows are not selected
             var group_ids = rowData['group_ids'].split(',').map(Number);
             group_ids.forEach(function (id) {
                 var index = $.inArray(id, history_to_delete);
                 if (index == -1) {
-                    $(row).find('button[data-id="' + rowData['id'] + '"]').addClass('btn-warning').removeClass('btn-danger');
+                    $(row).find('button[data-id="' + rowData['row_id'] + '"]').addClass('btn-warning').removeClass('btn-danger');
                 }
             });
         }
@@ -353,7 +353,7 @@ $('.history_table').on('click', '> tbody > tr > td.modal-control', function () {
     var rowData = row.data();
 
     $.get('get_stream_data', {
-        row_id: rowData['id'],
+        row_id: rowData['row_id'],
         session_key: rowData['session_key'],
         user: rowData['friendly_name']
     }).then(function (jqXHR) {
@@ -382,9 +382,9 @@ $('.history_table').on('click', '> tbody > tr > td.delete-control > button', fun
 
     if (rowData['group_count'] == 1) {
         // if no grouped rows simply add or remove row from history_to_delete
-        var index = $.inArray(rowData['id'], history_to_delete);
+        var index = $.inArray(rowData['row_id'], history_to_delete);
         if (index === -1) {
-            history_to_delete.push(rowData['id']);
+            history_to_delete.push(rowData['row_id']);
         } else {
             history_to_delete.splice(index, 1);
         }
@@ -549,7 +549,7 @@ function createChildTable(row, rowData) {
         var childRowData = childRow.data();
 
         $.get('get_stream_data', {
-            row_id: childRowData['id'],
+            row_id: childRowData['row_id'],
             user: childRowData['friendly_name']
         }).then(function (jqXHR) {
             $("#info-modal").html(jqXHR);
@@ -576,9 +576,9 @@ function createChildTable(row, rowData) {
         var childRowData = childRow.data();
 
         // add or remove row from history_to_delete
-        var index = $.inArray(childRowData['id'], history_to_delete);
+        var index = $.inArray(childRowData['row_id'], history_to_delete);
         if (index === -1) {
-            history_to_delete.push(childRowData['id']);
+            history_to_delete.push(childRowData['row_id']);
         } else {
             history_to_delete.splice(index, 1);
         }

data/interfaces/default/js/tables/history_table_modal.js

@@ -169,7 +169,7 @@ $('.history_table').on('click', 'td.modal-control', function () {
     function showStreamDetails() {
         $.ajax({
             url: 'get_stream_data',
-            data: { row_id: rowData['id'], user: rowData['friendly_name'] },
+            data: { row_id: rowData['row_id'], user: rowData['friendly_name'] },
             cache: false,
             async: true,
             complete: function (xhr, status) {

data/interfaces/default/js/tables/libraries.js

@@ -27,8 +27,8 @@ libraries_list_table_options = {
             "data": null,
             "createdCell": function (td, cellData, rowData, row, col) {
                 $(td).html('<div class="edit-library-toggles">' +
-                    '<button class="btn btn-xs btn-warning delete-library" data-id="' + rowData['section_id'] + '" data-toggle="button"><i class="fa fa-trash-o fa-fw"></i> Delete</button>&nbsp' +
-                    '<button class="btn btn-xs btn-warning purge-library" data-id="' + rowData['section_id'] + '" data-toggle="button"><i class="fa fa-eraser fa-fw"></i> Purge</button>&nbsp&nbsp&nbsp' +
+                    '<button class="btn btn-xs btn-warning delete-library" data-id="' + rowData['row_id'] + '" data-toggle="button"><i class="fa fa-trash-o fa-fw"></i> Delete</button>&nbsp' +
+                    '<button class="btn btn-xs btn-warning purge-library" data-id="' + rowData['row_id'] + '" data-toggle="button"><i class="fa fa-eraser fa-fw"></i> Purge</button>&nbsp&nbsp&nbsp' +
                     '<input type="checkbox" id="keep_history-' + rowData['section_id'] + '" name="keep_history" value="1" ' + rowData['keep_history'] + '><label class="edit-tooltip" for="keep_history-' + rowData['section_id'] + '" data-toggle="tooltip" title="Toggle History"><i class="fa fa-history fa-lg fa-fw"></i></label>&nbsp' +
                     '</div>');
             },
@@ -41,14 +41,16 @@ libraries_list_table_options = {
             "targets": [1],
             "data": "library_thumb",
             "createdCell": function (td, cellData, rowData, row, col) {
+                var inactive = '';
+                if (!rowData['is_active']) { inactive = '<span class="inactive-library-tooltip" data-toggle="tooltip" title="Library not on Plex server"><i class="fa fa-exclamation-triangle"></i></span>'; }
                 if (cellData !== null && cellData !== '') {
                     if (rowData['library_thumb'].substring(0, 4) == "http") {
-                        $(td).html('<a href="library?section_id=' + rowData['section_id'] + '"><div class="libraries-poster-face" style="background-image: url(' + rowData['library_thumb'] + ');"></div></a>');
+                        $(td).html('<a href="' + page('library', rowData['section_id']) + '"><div class="libraries-poster-face" style="background-image: url(' + rowData['library_thumb'] + ');">' + inactive + '</div></a>');
                     } else {
-                        $(td).html('<a href="library?section_id=' + rowData['section_id'] + '"><div class="libraries-poster-face svg-icon library-' + rowData['section_type'] + '"></div></a>');
+                        $(td).html('<a href="' + page('library', rowData['section_id']) + '"><div class="libraries-poster-face svg-icon library-' + rowData['section_type'] + '">' + inactive + '</div></a>');
                     }
                 } else {
-                    $(td).html('<a href="library?section_id=' + rowData['section_id'] + '"><div class="libraries-poster-face" style="background-image: url(../../images/cover.png);"></div></a>');
+                    $(td).html('<a href="' + page('library', rowData['section_id']) + '"><div class="libraries-poster-face" style="background-image: url(../../images/cover.png);">' + inactive + '</div></a>');
                 }
             },
             "orderable": false,
@@ -61,8 +63,8 @@ libraries_list_table_options = {
             "data": "section_name",
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData !== null && cellData !== '') {
-                    $(td).html('<div data-id="' + rowData['section_id'] + '">' +
-                        '<a href="library?section_id=' + rowData['section_id'] + '">' + cellData + '</a>' +
+                    $(td).html('<div data-id="' + rowData['row_id'] + '">' +
+                        '<a href="' + page('library', rowData['section_id']) + '">' + cellData + '</a>' +
                         '</div>');
                 } else {
                     $(td).html('n/a');
@@ -232,11 +234,11 @@ libraries_list_table_options = {
         showMsg(msg, false, false, 0)
     },
     "rowCallback": function (row, rowData) {
-        if ($.inArray(rowData['section_id'], libraries_to_delete) !== -1) {
-            $(row).find('button.delete-library[data-id="' + rowData['section_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
+        if ($.inArray(rowData['row_id'], libraries_to_delete) !== -1) {
+            $(row).find('button.delete-library[data-id="' + rowData['row_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
         }
-        if ($.inArray(rowData['section_id'], libraries_to_purge) !== -1) {
-            $(row).find('button.purge-library[data-id="' + rowData['section_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
+        if ($.inArray(rowData['row_id'], libraries_to_purge) !== -1) {
+            $(row).find('button.purge-library[data-id="' + rowData['row_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
         }
     }
 }
@@ -277,11 +279,11 @@ $('#libraries_list_table').on('click', 'td.edit-control > .edit-library-toggles
     var row = libraries_list_table.row(tr);
     var rowData = row.data();
 
-    var index_delete = $.inArray(rowData['section_id'], libraries_to_delete);
-    var index_purge = $.inArray(rowData['section_id'], libraries_to_purge);
+    var index_delete = $.inArray(rowData['row_id'], libraries_to_delete);
+    var index_purge = $.inArray(rowData['row_id'], libraries_to_purge);
 
     if (index_delete === -1) {
-        libraries_to_delete.push(rowData['section_id']);
+        libraries_to_delete.push(rowData['row_id']);
         if (index_purge === -1) {
             tr.find('button.purge-library').click();
         }
@@ -300,11 +302,11 @@ $('#libraries_list_table').on('click', 'td.edit-control > .edit-library-toggles
     var row = libraries_list_table.row(tr);
     var rowData = row.data();
 
-    var index_delete = $.inArray(rowData['section_id'], libraries_to_delete);
-    var index_purge = $.inArray(rowData['section_id'], libraries_to_purge);
+    var index_delete = $.inArray(rowData['row_id'], libraries_to_delete);
+    var index_purge = $.inArray(rowData['row_id'], libraries_to_purge);
 
     if (index_purge === -1) {
-        libraries_to_purge.push(rowData['section_id']);
+        libraries_to_purge.push(rowData['row_id']);
     } else {
         libraries_to_purge.splice(index_purge, 1);
         if (index_delete != -1) {

data/interfaces/default/js/tables/sync_table.js

@@ -51,7 +51,7 @@ sync_table_options = {
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData !== '') {
                     if (rowData['user_id']) {
-                        $(td).html('<a href="' + page('user', rowData['user_id']) + '>' + cellData + '</a>');
+                        $(td).html('<a href="' + page('user', rowData['user_id']) + '">' + cellData + '</a>');
                     } else {
                         $(td).html('<a href="' + page('user', null, rowData['user']) + '">' + cellData + '</a>');
                     }

data/interfaces/default/js/tables/user_ips.js

@@ -167,7 +167,7 @@ $('.user_ip_table').on('click', 'td.modal-control', function () {
     function showStreamDetails() {
         $.ajax({
             url: 'get_stream_data',
-            data: { row_id: rowData['id'], user: rowData['friendly_name'] },
+            data: { row_id: rowData['history_row_id'], user: rowData['friendly_name'] },
             cache: false,
             async: true,
             complete: function (xhr, status) {

data/interfaces/default/js/tables/users.js

@@ -44,8 +44,8 @@ users_list_table_options = {
             "data": null,
             "createdCell": function (td, cellData, rowData, row, col) {
                 $(td).html('<div class="edit-user-toggles">' + 
-                    '<button class="btn btn-xs btn-warning delete-user" data-id="' + rowData['user_id'] + '" data-toggle="button"><i class="fa fa-trash-o fa-fw"></i> Delete</button>&nbsp' +
-                    '<button class="btn btn-xs btn-warning purge-user" data-id="' + rowData['user_id'] + '" data-toggle="button"><i class="fa fa-eraser fa-fw"></i> Purge</button>&nbsp&nbsp&nbsp' +
+                    '<button class="btn btn-xs btn-warning delete-user" data-id="' + rowData['row_id'] + '" data-toggle="button"><i class="fa fa-trash-o fa-fw"></i> Delete</button>&nbsp' +
+                    '<button class="btn btn-xs btn-warning purge-user" data-id="' + rowData['row_id'] + '" data-toggle="button"><i class="fa fa-eraser fa-fw"></i> Purge</button>&nbsp&nbsp&nbsp' +
                     '<input type="checkbox" id="keep_history-' + rowData['user_id'] + '" name="keep_history" value="1" ' + rowData['keep_history'] + '><label class="edit-tooltip" for="keep_history-' + rowData['user_id'] + '" data-toggle="tooltip" title="Toggle History"><i class="fa fa-history fa-lg fa-fw"></i></label>&nbsp' +
                     '<input type="checkbox" id="allow_guest-' + rowData['user_id'] + '" name="allow_guest" value="1" ' + rowData['allow_guest'] + '><label class="edit-tooltip" for="allow_guest-' + rowData['user_id'] + '" data-toggle="tooltip" title="Toggle Guest Access"><i class="fa fa-unlock-alt fa-lg fa-fw"></i></label>&nbsp' +
                     '</div>');
@@ -59,10 +59,12 @@ users_list_table_options = {
             "targets": [1],
             "data": "user_thumb",
             "createdCell": function (td, cellData, rowData, row, col) {
+                var inactive = '';
+                if (!rowData['is_active']) { inactive = '<span class="inactive-user-tooltip" data-toggle="tooltip" title="User not on Plex server"><i class="fa fa-exclamation-triangle"></i></span>'; }
                 if (cellData === '') {
-                    $(td).html('<a href="' + page('user', rowData['user_id']) + '"><div class="users-poster-face" style="background-image: url(../../images/gravatar-default-80x80.png);"></div></a>');
+                    $(td).html('<a href="' + page('user', rowData['user_id']) + '"><div class="users-poster-face" style="background-image: url(../../images/gravatar-default-80x80.png);">' + inactive + '</div></a>');
                 } else {
-                    $(td).html('<a href="' + page('user', rowData['user_id']) + '"><div class="users-poster-face" style="background-image: url(' + rowData['user_thumb'] + ');"></div></a>');
+                    $(td).html('<a href="' + page('user', rowData['user_id']) + '"><div class="users-poster-face" style="background-image: url(' + rowData['user_thumb'] + ');">' + inactive + '</div></a>');
                 }
             },
             "orderable": false,
@@ -75,7 +77,7 @@ users_list_table_options = {
             "data": "friendly_name",
             "createdCell": function (td, cellData, rowData, row, col) {
                 if (cellData !== null && cellData !== '') {
-                    $(td).html('<div class="edit-user-name" data-id="' + rowData['user_id'] + '">' +
+                    $(td).html('<div class="edit-user-name" data-id="' + rowData['row_id'] + '">' +
                         '<a href="' + page('user', rowData['user_id']) + '">' + cellData + '</a>' +
                         '<input type="text" class="hidden" value="' + cellData + '">' +
                         '</div>');
@@ -254,10 +256,10 @@ users_list_table_options = {
     },
     "rowCallback": function (row, rowData) {
         if ($.inArray(rowData['user_id'], users_to_delete) !== -1) {
-            $(row).find('button.delete-user[data-id="' + rowData['user_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
+            $(row).find('button.delete-user[data-id="' + rowData['row_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
         }
         if ($.inArray(rowData['user_id'], users_to_purge) !== -1) {
-            $(row).find('button.purge-user[data-id="' + rowData['user_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
+            $(row).find('button.purge-user[data-id="' + rowData['row_id'] + '"]').toggleClass('btn-warning').toggleClass('btn-danger');
         }
     }
 }
@@ -268,7 +270,7 @@ $('#users_list_table').on('click', 'td.modal-control', function () {
     var rowData = row.data();
 
     $.get('get_stream_data', {
-        row_id: rowData['id'],
+        row_id: rowData['history_row_id'],
         user: rowData['friendly_name']
     }).then(function (jqXHR) {
         $("#info-modal").html(jqXHR);
@@ -326,11 +328,11 @@ $('#users_list_table').on('click', 'td.edit-control > .edit-user-toggles > butto
     var row = users_list_table.row(tr);
     var rowData = row.data();
 
-    var index_delete = $.inArray(rowData['user_id'], users_to_delete);
-    var index_purge = $.inArray(rowData['user_id'], users_to_purge);
+    var index_delete = $.inArray(rowData['row_id'], users_to_delete);
+    var index_purge = $.inArray(rowData['row_id'], users_to_purge);
 
     if (index_delete === -1) {
-        users_to_delete.push(rowData['user_id']);
+        users_to_delete.push(rowData['row_id']);
         if (index_purge === -1) {
             tr.find('button.purge-user').click();
         }
@@ -349,11 +351,11 @@ $('#users_list_table').on('click', 'td.edit-control > .edit-user-toggles > butto
     var row = users_list_table.row(tr);
     var rowData = row.data();
 
-    var index_delete = $.inArray(rowData['user_id'], users_to_delete);
-    var index_purge = $.inArray(rowData['user_id'], users_to_purge);
+    var index_delete = $.inArray(rowData['row_id'], users_to_delete);
+    var index_purge = $.inArray(rowData['row_id'], users_to_purge);
 
     if (index_purge === -1) {
-        users_to_purge.push(rowData['user_id']);
+        users_to_purge.push(rowData['row_id']);
     } else {
         users_to_purge.splice(index_purge, 1);
         if (index_delete != -1) {

data/interfaces/default/libraries.html

@@ -116,14 +116,14 @@
                     });
 
                     if (libraries_to_delete.length > 0) {
-                        $('#libraries-to-delete').prepend('<p>Are you REALLY sure you want to delete the following libraries:</p>')
+                        $('#libraries-to-delete').prepend('<p>Are you REALLY sure you want to delete the following libraries:</p>');
                         for (var i = 0; i < libraries_to_delete.length; i++) {
                             $('#libraries-to-delete').append('<li>' + $('div[data-id=' + libraries_to_delete[i] + ']').text() + '</li>');
                         }
                     }
 
                     if (libraries_to_purge.length > 0) {
-                        $('#libraries-to-purge').prepend('<p>Are you REALLY sure you want to purge all history for the following libraries:</p>')
+                        $('#libraries-to-purge').prepend('<p>Are you REALLY sure you want to purge all history for the following libraries:</p>');
                         for (var i = 0; i < libraries_to_purge.length; i++) {
                             $('#libraries-to-purge').append('<li>' + $('div[data-id=' + libraries_to_purge[i] + ']').text() + '</li>');
                         }
@@ -131,33 +131,30 @@
 
                     $('#confirm-modal-delete').modal();
                     $('#confirm-modal-delete').one('click', '#confirm-delete', function () {
-                        libraries_to_delete.forEach(function(row, idx) {
-                            $.ajax({
-                                url: 'delete_library',
-                                type: 'POST',
-                                data: { section_id: row },
-                                cache: false,
-                                async: true,
-                                success: function (data) {
-                                    var msg = "Library deleted";
-                                    showMsg(msg, false, true, 2000);
-                                }
-                            });
+                        $.ajax({
+                            url: 'delete_all_library_history',
+                            type: 'POST',
+                            data: { row_ids: libraries_to_purge.join(',') },
+                            cache: false,
+                            async: true,
+                            success: function (data) {
+                                var msg = "Library history purged";
+                                showMsg(msg, false, true, 2000);
+                                libraries_list_table.draw();
+                            }
                         });
-                        libraries_to_purge.forEach(function(row, idx) {
-                            $.ajax({
-                                url: 'delete_all_library_history',
-                                type: 'POST',
-                                data: { section_id: row },
-                                cache: false,
-                                async: true,
-                                success: function (data) {
-                                    var msg = "Library history purged";
-                                    showMsg(msg, false, true, 2000);
-                                }
-                            });
+                        $.ajax({
+                            url: 'delete_library',
+                            type: 'POST',
+                            data: { row_ids: libraries_to_delete.join(',') },
+                            cache: false,
+                            async: true,
+                            success: function (data) {
+                                var msg = "Library deleted";
+                                showMsg(msg, false, true, 2000);
+                                libraries_list_table.draw();
+                            }
                         });
-                        libraries_list_table.draw();
                     });
                 }
 
@@ -188,7 +185,7 @@
             complete: function (xhr, status) {
                 var result = $.parseJSON(xhr.responseText);
                 var msg = result.message;
-                if (result.result == 'success') {
+                if (result.result === 'success') {
                     showMsg('<i class="fa fa-check"></i> ' + msg, false, true, 2000, false);
                     libraries_list_table.draw();
                 } else {

data/interfaces/default/library.html

@@ -61,10 +61,22 @@ DOCUMENTATION :: END
                 <div class="col-md-12">
                     <div class="table-card-back">
                         <div class="user-info-wrapper">
-                            % if data['library_thumb'][:4] == 'http':
-                            <div class="library-info-poster-face" style="background-image: url(${data['library_thumb']});"></div>
+                            % if data['library_thumb'].startswith('http'):
+                            <div class="library-info-poster-face" style="background-image: url(${page('pms_image_proxy', data['library_thumb'], None, 80, 80)});">
+                                % if not data['is_active']:
+                                <span class="inactive-library-tooltip" data-toggle="tooltip" title="Library not on Plex server">
+                                    <i class="fa fa-2x fa-exclamation-triangle"></i>
+                                </span>
+                                % endif
+                            </div>
                             % else:
-                            <div class="library-info-poster-face svg-icon library-${data['section_type']}"></div>
+                            <div class="library-info-poster-face svg-icon library-${data['section_type']}">
+                                % if not data['is_active']:
+                                <span class="inactive-library-tooltip" data-toggle="tooltip" title="Library not on Plex server">
+                                    <i class="fa fa-2x fa-exclamation-triangle"></i>
+                                </span>
+                                % endif
+                            </div>
                             % endif
                             <div class="user-info-username">
                                 <span class="set-username">${data['section_name']}</span>
@@ -239,9 +251,9 @@ DOCUMENTATION :: END
                                     % else:
                                     <div id="get_file_sizes_message" style="text-align: center; margin-top: 20px; display: none;">
                                     % endif
-                                        <i class="fa fa-refresh fa-spin"></i> Tautulli is calculating the file sizes for the library's media info. This could take a few minutes depending on the size of your library.
+                                        <i class="fa fa-refresh fa-spin"></i>&nbsp; Tautulli is calculating the file sizes for the library's media info. This could take a few minutes depending on the size of your library.
                                         <br />
-                                        You may leave this page and come back later.
+                                        You may leave this page and check back later.
                                     </div>
                                     <div class='table-card-header'>
                                         <div class="header-bar">
@@ -411,6 +423,8 @@ DOCUMENTATION :: END
         history_table.draw();
     });
 
+    $(".inactive-library-tooltip").tooltip();
+
     % if _session['user_group'] == 'admin':
     function loadMediaInfoTable() {
         // Build media info table
@@ -471,19 +485,17 @@ DOCUMENTATION :: END
                 $('#deleteCount').text(history_to_delete.length);
                 $('#confirm-modal-delete').modal();
                 $('#confirm-modal-delete').one('click', '#confirm-delete', function () {
-                    history_to_delete.forEach(function(row, idx) {
-                        $.ajax({
-                            url: 'delete_history_rows',
-                            type: 'POST',
-                            data: { row_id: row },
-                            async: true,
-                            success: function (data) {
-                                var msg = "History deleted";
-                                showMsg(msg, false, true, 2000);
-                            }
-                        });
+                    $.ajax({
+                        url: 'delete_history_rows',
+                        type: 'POST',
+                        data: { row_ids: history_to_delete.join(',') },
+                        async: true,
+                        success: function (data) {
+                            var msg = "History deleted";
+                            showMsg(msg, false, true, 2000);
+                            history_table.draw();
+                        }
                     });
-                    history_table.draw();
                 });
             }
 

data/interfaces/default/library_stats.html

@@ -35,10 +35,17 @@ DOCUMENTATION :: END
 %>
 % for section_type in types:
 % if section_type in data:
+<%
+    row0 = data[section_type][0]
+%>
 <div class="dashboard-stats-instance" id="library-stats-instance-${section_type}" data-section_type="${section_type}">
     <div class="dashboard-stats-container">
-        <div id="library-stats-background-${section_type}" class="dashboard-stats-background" style="background-image: url(${page('pms_image_proxy', '/:/resources/' + section_type + '-fanart.jpg', None, 500, 280, 40, '282828', 3, fallback='art')});">
+        <div id="library-stats-background-${section_type}" class="dashboard-stats-background" style="background-image: url(${page('pms_image_proxy', row0['art'], None, 500, 280, 40, '282828', 3, fallback='art')});">
+            % if row0['thumb'].startswith('http'):
+            <div id="library-stats-thumb-${section_type}" class="dashboard-stats-flat hidden-xs" style="background-image: url(${page('pms_image_proxy', row0['thumb'], None, 80, 80)});"></div>
+            % else:
             <div id="library-stats-thumb-${section_type}" class="dashboard-stats-flat svg-icon library-${section_type} hidden-xs"></div>
+            % endif
             <div class="dashboard-stats-info-container">
                 <div id="library-stats-title-${section_type}" class="dashboard-stats-info-title">
                     <h4>${headers[section_type][0]}</h4>
@@ -48,7 +55,8 @@ DOCUMENTATION :: END
                     <div class="dashboard-stats-info scoller-content">
                         <ul class="list-unstyled dashboard-stats-info-list">
                             % for section in data[section_type]:
-                            <li class="dashboard-stats-info-item ${'expanded' if loop.index == 0 else ''}">
+                            <li class="dashboard-stats-info-item ${'expanded' if loop.index == 0 else ''}" data-stat_id="${section_type}"
+                                data-art="${section.get('art')}" data-thumb="${section.get('thumb')}">
                                 <div class="sub-list">${loop.index + 1}</div>
                                 <div class="sub-value">
                                     <a href="${page('library', section['section_id'])}" title="${section['section_name']}">

data/interfaces/default/login.html

@@ -24,7 +24,7 @@
 
     <!-- ICONS -->
     <!-- Android -->
-    <link rel="manifest" href="${http_root}images/favicon/manifest.json?v=2.0.5" crossorigin="use-credentials>
+    <link rel="manifest" href="${http_root}images/favicon/manifest.json?v=2.0.5" crossorigin="use-credentials">
     <meta name="theme-color" content="#282a2d">
     <!-- Apple -->
     <link rel="apple-touch-icon" sizes="180x180" href="${http_root}images/favicon/apple-touch-icon.png?v=2.0.5">

data/interfaces/default/mobile_devices_table.html

@@ -13,7 +13,11 @@ DOCUMENTATION :: END
     % for device in sorted(devices_list, key=lambda k: k['device_name']):
     <li class="mobile-device pointer" data-id="${device['id']}" data-name="${device['device_name']}">
         <span>
-            <span class="toggle-left"><i class="fa fa-lg fa-fw fa-mobile"></i></span>
+            % if device['official']:
+                <span class="toggle-left"><i class="fa fa-lg fa-fw fa-mobile"></i></span>
+            % else:
+                <span class="toggle-left officail-tooltip" data-toggle="tooltip" data-placement="top" title="Unofficial or Unknown App"><i class="fa fa-lg fa-fw fa-exclamation-triangle"></i></span>
+            % endif
             ${device['friendly_name'] or device['device_name']} &nbsp;<span class="friendly_name">(${device['id']})</span>
             <span class="toggle-right"><i class="fa fa-lg fa-fw fa-cog"></i></span>
             <span class="toggle-right friendly_name" id="device-last_seen-${device['id']}">
@@ -138,4 +142,6 @@ DOCUMENTATION :: END
         }
         verifiedDevice = true;
     })
+
+    $('.officail-tooltip').tooltip();
 </script>

data/interfaces/default/newsletter_config.html

@@ -123,7 +123,7 @@
                                         <div class="row">
                                             <div class="col-md-12">
                                                 <select class="form-control" id="${item['name']}" name="${item['name']}">
-                                                    % for key, value in sorted(item['select_options'].iteritems()):
+                                                    % for key, value in sorted(item['select_options'].items()):
                                                     % if key == item['value']:
                                                     <option value="${key}" selected>${value}</option>
                                                     % else:
@@ -144,7 +144,7 @@
                                                     <option value="select-all">Select All</option>
                                                     <option value="remove-all">Remove All</option>
                                                     % if isinstance(item['select_options'], dict):
-                                                    % for section, options in item['select_options'].iteritems():
+                                                    % for section, options in item['select_options'].items():
                                                     <optgroup label="${section}">
                                                         % for option in sorted(options, key=lambda x: x['text'].lower()):
                                                             <option value="${option['value']}">${option['text']}</option>
@@ -325,7 +325,7 @@
                                             <div class="row">
                                                 <div class="col-md-12">
                                                     <select class="form-control" id="${item['name']}" name="${item['name']}">
-                                                        % for key, value in sorted(item['select_options'].iteritems()):
+                                                        % for key, value in sorted(item['select_options'].items()):
                                                         % if key == item['value']:
                                                         <option value="${key}" selected>${value}</option>
                                                         % else:
@@ -346,7 +346,7 @@
                                                         <option value="select-all">Select All</option>
                                                         <option value="remove-all">Remove All</option>
                                                         % if isinstance(item['select_options'], dict):
-                                                        % for section, options in item['select_options'].iteritems():
+                                                        % for section, options in item['select_options'].items():
                                                         <optgroup label="${section}">
                                                             % for option in sorted(options, key=lambda x: x['text'].lower()):
                                                                 <option value="${option['value']}">${option['text']}</option>

data/interfaces/default/newsletter_preview.html

@@ -1,5 +1,5 @@
 <%
-    import urllib
+    from six.moves.urllib.parse import urlencode
 %>
 <!doctype html>
 
@@ -8,6 +8,9 @@
     <meta charset="utf-8">
     <title>Tautulli - ${title} | ${server_name}</title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="icon" type="image/png" sizes="32x32" href="${http_root}images/favicon/favicon-32x32.png?v=2.0.5">
+    <link rel="icon" type="image/png" sizes="16x16" href="${http_root}images/favicon/favicon-16x16.png?v=2.0.5">
+    <link rel="shortcut icon" href="${http_root}images/favicon/favicon.ico?v=2.0.5">
     <link href="${http_root}css/tautulli.css${cache_param}" rel="stylesheet">
     <style>
         * {
@@ -32,7 +35,7 @@
 <script>
 $(document).ready(function () {
     var frame = $('<iframe></iframe>', {
-        src: 'real_newsletter?${urllib.urlencode(kwargs) | n}',
+        src: 'real_newsletter?${urlencode(kwargs) | n}',
         frameborder: '0',
         style: 'display: none; height: 100vh; width: 100vw;'
     });

data/interfaces/default/newsletters_table.html

@@ -9,7 +9,7 @@ Version:            0.1
 DOCUMENTATION :: END
 </%doc>
 
-<% from plexpy.newsletter_handler import NEWSLETTER_SCHED %>
+<% from plexpy import newsletter_handler %>
 <ul class="stacked-configs list-unstyled">
     % for newsletter in sorted(newsletters_list, key=lambda k: (k['agent_label'], k['friendly_name'], k['id'])):
     <li class="newsletter-agent pointer" data-id="${newsletter['id']}">
@@ -22,8 +22,8 @@ DOCUMENTATION :: END
             % endif
             <span class="toggle-right"><i class="fa fa-lg fa-fw fa-cog"></i></span>
             <span class="toggle-right friendly_name" id="newsletter-next_run-${newsletter['id']}">
-                % if NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])):
-                <% job = NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])) %>
+                % if newsletter_handler.NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])):
+                <% job = newsletter_handler.NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])) %>
                 <script>
                     $("#newsletter-next_run-${newsletter['id']}").text(moment("${job.next_run_time}", "YYYY-MM-DD HH:mm:ssZ").fromNow())
                 </script>

data/interfaces/default/notifier_config.html

@@ -1,9 +1,9 @@
 % if notifier:
-<%!
+<%
     import json
     from plexpy import notifiers, users
     from plexpy.helpers import checked
-    available_notification_actions = notifiers.available_notification_actions()
+    available_notification_actions = notifiers.available_notification_actions(agent_id=notifier['agent_id'])
 
     user_emails = [{'user': u['friendly_name'] or u['username'], 'email': u['email']} for u in users.Users().get_users() if u['email']]
     sorted(user_emails, key=lambda u: u['user'])
@@ -25,7 +25,7 @@
                         <li role="presentation"><a href="#tabs-notify_text" aria-controls="tabs-notify_text" role="tab" data-toggle="tab">Arguments</a></li>
                         % elif notifier['agent_name'] == 'webhook':
                         <li role="presentation"><a href="#tabs-notify_text" aria-controls="tabs-notify_text" role="tab" data-toggle="tab">Data</a></li>
-                        % else:
+                        % elif notifier['agent_name'] != 'plexmobileapp':
                         <li role="presentation"><a href="#tabs-notify_text" aria-controls="tabs-notify_text" role="tab" data-toggle="tab">Text</a></li>
                         % endif
                         <li role="presentation"><a href="#tabs-test_notifications" aria-controls="tabs-test_notifications" role="tab" data-toggle="tab">Test Notifications</a></li>
@@ -88,7 +88,7 @@
                                         <div class="row">
                                             <div class="col-md-12">
                                                 <select class="form-control" id="${item['name']}" name="${item['name']}">
-                                                    % for key, value in sorted(item['select_options'].iteritems()):
+                                                    % for key, value in sorted(item['select_options'].items()):
                                                     % if key == item['value']:
                                                     <option value="${key}" selected>${value}</option>
                                                     % else:
@@ -109,7 +109,7 @@
                                                     <option value="select-all">Select All</option>
                                                     <option value="remove-all">Remove All</option>
                                                     % if isinstance(item['select_options'], dict):
-                                                    % for section, options in item['select_options'].iteritems():
+                                                    % for section, options in item['select_options'].items():
                                                     <optgroup label="${section}">
                                                         % for option in sorted(options, key=lambda x: x['text'].lower()):
                                                             <option value="${option['value']}">${option['text']}</option>
@@ -211,7 +211,7 @@
                                 % for action in available_notification_actions:
                                 <li>
                                     <div class="link">
-                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>&nbsp;
+                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>
                                         ${action['label']}
                                         <span class="toggle-right"><i class="fa fa-chevron-down"></i></span>
                                     </div>
@@ -237,7 +237,7 @@
                                 % for action in available_notification_actions:
                                 <li>
                                     <div class="link">
-                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>&nbsp;
+                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>
                                         ${action['label']}
                                         <span class="toggle-right"><i class="fa fa-chevron-down"></i></span>
                                     </div>
@@ -268,7 +268,7 @@
                                 % for action in available_notification_actions:
                                 <li>
                                     <div class="link">
-                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>&nbsp;
+                                        <span class="toggle-left"><i class="fa ${action['icon']} fa-fw"></i></span>
                                         ${action['label']}
                                         <span class="toggle-right"><i class="fa fa-chevron-down"></i></span>
                                     </div>
@@ -313,7 +313,7 @@
                                 <div class="row">
                                     <div class="col-md-12">
                                         <select class="form-control" id="test_script" name="test_script">
-                                            % for key, value in sorted(notifier['config_options'][2]['select_options'].iteritems()):
+                                            % for key, value in sorted(notifier['config_options'][2]['select_options'].items()):
                                             <option value="${key}">${value}</option>
                                             % endfor
                                         </select>
@@ -684,6 +684,15 @@
         pushoverPriority();
     });
 
+    % elif notifier['agent_name'] == 'plexmobileapp':
+    var $plexmobileapp_user_ids = $('#plexmobileapp_user_ids').selectize({
+        plugins: ['remove_button'],
+        maxItems: null,
+        create: true
+    });
+    var plexmobileapp_user_ids = $plexmobileapp_user_ids[0].selectize;
+    plexmobileapp_user_ids.setValue(${json.dumps(next((c['value'] for c in notifier['config_options'] if c['name'] == 'plexmobileapp_user_ids'), [])) | n});
+
     % endif
 
     function validateLogic() {
@@ -844,10 +853,8 @@
                 PNotify.prototype.options.hide = true;
                 PNotify.prototype.options.delay = $('#browser_auto_hide_delay').val() * 1000;
             }
-            var notification = new PNotify({
-                title: $('#test_subject').val(),
-                text: $('#test_body').val()
-            });
+            displayPNotify($('#test_subject').val(), $('#test_body').val());
+            showMsg('<i class="fa fa-check"></i> Notification sent.', false, true, 5000);
         }
     }
 

data/interfaces/default/settings.html

@@ -8,7 +8,7 @@
     from plexpy.helpers import anon_url, checked
 
     docker_setting = 'disabled' if plexpy.DOCKER else ''
-    docker_msg = '<span class="docker-setting small">(Controlled by Docker Container)</span>' if plexpy.DOCKER else ''
+    docker_msg = '<span class="setting-message small">(Controlled by Docker Container)</span>' if plexpy.DOCKER else ''
 
     available_notification_agents = sorted(notifiers.available_notification_agents(), key=lambda k: k['label'].lower())
     available_newsletter_agents = sorted(newsletters.available_newsletter_agents(), key=lambda k: k['label'].lower())
@@ -71,6 +71,13 @@
                             <h3>Version ${common.RELEASE} <small><a id="changelog-modal-link" href="#"><i class="fa fa-info-circle"></i> Changelog</a></small></h3>
                         </div>
                         % endif
+                        <div class="padded-header">
+                            <h3>Tautulli News</h3>
+                        </div>
+                        <div id="tautulli-news">
+                            <div class='text-muted'><i class="fa fa-refresh fa-spin"></i> Loading news...</div>
+                            <br>
+                        </div>
                         <div class="padded-header">
                             <h3>Tautulli Configuration</h3>
                         </div>
@@ -215,12 +222,14 @@
                             <p class="help-block">Check for Tautulli updates periodically.</p>
                         </div>
                         <div id="git_update_options">
+                            % if not plexpy.FROZEN:
                             <div class="checkbox">
                                 <label>
                                     <input type="checkbox" id="plexpy_auto_update" name="plexpy_auto_update" value="1" ${config['plexpy_auto_update']} ${docker_setting}> Update Automatically ${docker_msg | n}
                                 </label>
                                 <p class="help-block">Update Tautulli automatically if an update is available.</p>
                             </div>
+                            % endif
                             <div class="form-group advanced-setting">
                                 <label for="git_token">GitHub API Token</label>
                                 <div class="row">
@@ -265,6 +274,20 @@
                             </div>
                             <p class="help-block">Optional: The path to your git environment variable. Leave blank for default.</p>
                         </div>
+                        <div class="form-group advanced-setting">
+                            <label>Repair Git Install</label>
+                            <p class="help-block">
+                                Attempt to fix updating by resetting your Tautulli installation back to <strong>${common.RELEASE}</strong>.<br />
+                                Note: This will not affect any saved history or settings.
+                            </p>
+                            <div class="row">
+                                <div class="col-md-4">
+                                    <div class="btn-group">
+                                        <button class="btn btn-form" type="button" id="reset_git_install">Reset</button>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
                         % endif
 
                         <p><input type="button" class="btn btn-bright save-button" value="Save" data-success="Changes saved successfully"></p>
@@ -434,12 +457,27 @@
                         </div>
 
                         <p class="help-block">Note: Web interface changes require a restart.</p>
-                        % if os.name == 'nt':
+                        % if common.PLATFORM in ('Windows', 'Darwin'):
+                        <%
+                            tray = {'Windows': 'System Tray', 'Darwin': 'Menu Bar'}
+                            tray_disabled = tray_disabled_msg = ''
+                            if common.PLATFORM == 'Darwin':
+                                from plexpy.macos import HAS_PYOBJC
+                                if not HAS_PYOBJC:
+                                    tray_disabled = 'disabled'
+                                    tray_disabled_msg = '<span class="setting-message small">(Missing pyobjc module)</span>'
+                        %>
                         <div class="checkbox">
                             <label>
-                                <input type="checkbox" class="http-settings" name="win_sys_tray" id="win_sys_tray" value="1" ${config['win_sys_tray']}> Enable System Tray Icon
+                                <input type="checkbox" class="http-settings" name="sys_tray_icon" id="sys_tray_icon" value="1" ${config['sys_tray_icon']} ${tray_disabled}> Enable ${tray[common.PLATFORM]} Icon ${tray_disabled_msg | n}
                             </label>
-                            <p class="help-block">Show Tautulli shortcut in the system tray.</p>
+                            <p class="help-block">Show Tautulli shortcut in the ${tray[common.PLATFORM].lower()}.</p>
+                        </div>
+                        <div class="checkbox">
+                            <label>
+                                <input type="checkbox" name="launch_startup" id="launch_startup" value="1" ${config['launch_startup']}> Launch at System Startup
+                            </label>
+                            <p class="help-block">Start Tautulli automatically after Login.</p>
                         </div>
                         % endif
                         <div class="checkbox">
@@ -836,6 +874,28 @@
                             <span id="remoteAccessCheck" class="settings-warning"></span>
                             <p class="help-block">Enable to have Tautulli check if remote access to the Plex Media Server goes down.</p>
                         </div>
+                        <div id="monitor_remote_access_options">
+                            <div class="form-group advanced-setting">
+                                <label for="remote_access_ping_interval">Remote Access Ping Interval</label>
+                                <div class="row">
+                                    <div class="col-md-2">
+                                        <input type="text" class="form-control" data-parsley-type="integer" id="remote_access_ping_interval" name="remote_access_ping_interval" value="${config['remote_access_ping_interval']}" size="5" data-parsley-min="60" data-parsley-trigger="change" data-parsley-errors-container="#remote_access_ping_interval_error" required>
+                                    </div>
+                                    <div id="remote_access_ping_interval_error" class="alert alert-danger settings-alert" role="alert"></div>
+                                </div>
+                                <p class="help-block">The interval (in seconds) Tautulli will ping the Plex Media Server for the remote access status. Minimum 60.</p>
+                            </div>
+                            <div class="form-group advanced-setting">
+                                <label for="remote_access_ping_threshold">Remote Access Ping Threshold</label>
+                                <div class="row">
+                                    <div class="col-md-2">
+                                        <input type="text" class="form-control" data-parsley-type="integer" id="remote_access_ping_threshold" name="remote_access_ping_threshold" value="${config['remote_access_ping_threshold']}" size="5" data-parsley-min="1" data-parsley-trigger="change" data-parsley-errors-container="#remote_access_ping_threshold_error" required>
+                                    </div>
+                                    <div id="remote_access_ping_threshold_error" class="alert alert-danger settings-alert" role="alert"></div>
+                                </div>
+                                <p class="help-block">The number of consecutive remote access status failures to consider remote access as down. Minimum 1.</p>
+                            </div>
+                        </div>
 
                         <div class="form-group advanced-setting">
                             <label for="refresh_users_interval">Users List Refresh Interval</label>
@@ -927,7 +987,7 @@
                                 </div>
                                 <div id="buffer_wait_error" class="alert alert-danger settings-alert" role="alert"></div>
                             </div>
-                            <p class="help-block">The value (in seconds) Tautulli should wait before triggering the next buffer warning. 0 to always trigger.</p>
+                            <p class="help-block">The value (in seconds) Tautulli should wait before triggering the next buffer warning. Set to 0 to always trigger.</p>
                         </div>
                         <div class="checkbox advanced-setting">
                             <label>
@@ -951,6 +1011,20 @@
                             </div>
                             <p class="help-block">The number of concurrent streams by a single user for Tautulli to trigger a notification. Minimum 2.</p>
                         </div>
+                        <div class="form-group advanced-setting">
+                            <label for="notify_concurrent_threshold">Continued Session Threshold</label>
+                            <div class="row">
+                                <div class="col-md-2">
+                                    <input type="text" class="form-control" data-parsley-type="integer" id="notify_continued_session_threshold" name="notify_continued_session_threshold" value="${config['notify_continued_session_threshold']}" data-parsley-min="0" data-parsley-trigger="change" data-parsley-errors-container="#notify_continued_session_threshold_error" required>
+                                </div>
+                                <div id="notify_continued_session_threshold_error" class="alert alert-danger settings-alert" role="alert"></div>
+                            </div>
+                            <p class="help-block">
+                                The number of seconds between stopping and starting a new stream to be considered as a continued session. Set to 0 to consider all streams as new sessions.
+                                <br>
+                                Note: The threshold is only used by the "Initial Stream" notification parameter to determine if a stream is the first stream of a continuous streaming session.
+                            </p>
+                        </div>
 
                         <div class="padded-header">
                             <h3>Recently Added Notifications</h3>
@@ -982,7 +1056,21 @@
                                 </div>
                                 <div id="notify_recently_added_delay_error" class="alert alert-danger settings-alert" role="alert"></div>
                             </div>
-                            <p class="help-block">Set the delay (in seconds) to wait for consecutive recently added items to group together and to allow metadata to be processed before sending the notification. Minimum 60 seconds.</p>
+                            <p class="help-block">Set the delay (in seconds) to wait for consecutive recently added items to group together and to allow metadata to be processed before sending the recently added notification. Minimum 60 seconds, default 300.</p>
+                        </div>
+                        <div class="form-group advanced-setting">
+                            <label>Flush Recently Added</label>
+                            <p class="help-block">
+                                Attempt to fix recently added notifications by flushing out all of the recently added items in the database.<br />
+                                Warning: This will reset all recently added notifications. For emergency use only when recently added notifications are stuck!
+                            </p>
+                            <div class="row">
+                                <div class="col-md-4">
+                                    <div class="btn-group">
+                                        <button class="btn btn-form" type="button" id="delete_recently_added">Flush</button>
+                                    </div>
+                                </div>
+                            </div>
                         </div>
                         <!--<div class="checkbox">
                             <label>
@@ -1225,53 +1313,18 @@
                             </label>
                             <p class="help-block">Enable to lookup links to MusicBrainz for music when available.</p>
                         </div>
-
-                        <div class="padded-header">
-                            <h3>Geolocation Database</h3>
-                        </div>
-
-                        <p class="help-block">The GeoLite2 database is used to geolocate IP addresses.</p>
-                        <p class="help-block">
-                            Please see the <a target='_blank' href='${anon_url('https://github.com/%s/%s-Wiki/wiki/3rd-Party-APIs-Guide' % (plexpy.CONFIG.GIT_USER, plexpy.CONFIG.GIT_REPO))}'>3rd Party APIs Guide</a> for instructions on setting up MaxMind.<br>
-                        </p>
                         <div class="form-group">
-                            <label for="maxmind_license_key">MaxMind License Key</label>
-                            <div class="row">
-                                <div class="col-md-6">
-                                    <input type="text" class="form-control" id="maxmind_license_key" name="maxmind_license_key" value="${config['maxmind_license_key']}" data-parsley-trigger="change">
-                                </div>
-                            </div>
-                            <p class="help-block">
-                                Enter your MaxMind License Key to install the GeoLite2 database.
-                            </p>
-                        </div>
-                        <div class="form-group">
-                            <label for="geoip_db">GeoLite2 Database File</label> ${docker_msg | n}
+                            <label for="delete_lookup_info">Delete Lookup Info</label>
+                            <p class="help-block">Delete all cached metadata lookup info in Tautulli.</p>
                             <div class="row">
                                 <div class="col-md-9">
-                                    <div class="input-group">
-                                        <input type="text" class="form-control" id="geoip_db" name="geoip_db" value="${config['geoip_db']}" ${docker_setting} data-parsley-trigger="change" data-parsley-pattern=".+\.mmdb$" data-parsley-errors-container="#geoip_db_error"  data-parsley-error-message="Must end with '.mmdb'">
-                                        <span class="input-group-btn">
-                                            <button class="btn btn-form" type="button" id="install_geoip_db">${'Update' if config["geoip_db_installed"] else 'Install'}</button>
-                                            <button class="btn btn-form" type="button" id="uninstall_geoip_db" ${'disabled' if not config['geoip_db_installed'] else ''}>Uninstall</button>
-                                        </span>
+                                    <div class="btn-group">
+                                        <button class="btn btn-form delete_all_lookups" type="button" data-service="themoviedb">TheMovieDB</button>
+                                        <button class="btn btn-form delete_all_lookups" type="button" data-service="tvmaze">TVmaze</button>
+                                        <button class="btn btn-form delete_all_lookups" type="button" data-service="musicbrainz">MusicBrainz</button>
                                     </div>
                                 </div>
-                                <div id="geoip_db_error" class="alert alert-danger settings-alert" role="alert"></div>
                             </div>
-                            <p class="help-block">
-                                Leave blank to install in the default location. GeoLite2 database last updated <strong><span id="geoip_db_updated">never</span></strong>.
-                            </p>
-                        </div>
-                        <div class="form-group advanced-setting">
-                            <label for="geoip_db_update_days">GeoLite2 Database Update Interval</label>
-                            <div class="row">
-                                <div class="col-md-2">
-                                    <input type="text" class="form-control" data-parsley-type="integer" id="geoip_db_update_days" name="geoip_db_update_days" value="${config['geoip_db_update_days']}" size="5" data-parsley-range="[7, 30]" data-parsley-trigger="change" data-parsley-errors-container="#geoip_db_update_days_error" required>
-                                </div>
-                                <div id="geoip_db_update_days_error" class="alert alert-danger settings-alert" role="alert"></div>
-                            </div>
-                            <p class="help-block">The interval (in days) Tautulli will automatically update the GeoLite2 database. Minimum 7, maximum 30, default 30.</p>
                         </div>
 
                         <p><input type="button" class="btn btn-bright save-button" value="Save" data-success="Changes saved successfully"></p>
@@ -1284,8 +1337,9 @@
                             <h3>Database Import</h3>
                         </div>
 
-                        <p class="help-block">Click a button below to import an existing database from another app.</p>
+                        <p class="help-block">Click a button below to import an existing database from the selected app.</p>
                         <div class="btn-group">
+                            <button class="btn btn-form toggle-app-import-modal" type="button" data-target="#app-import-modal" data-toggle="modal" data-app="tautulli">Tautulli</button>
                             <button class="btn btn-form toggle-app-import-modal" type="button" data-target="#app-import-modal" data-toggle="modal" data-app="plexwatch">PlexWatch</button>
                             <button class="btn btn-form toggle-app-import-modal" type="button" data-target="#app-import-modal" data-toggle="modal" data-app="plexivity">Plexivity</button>
                         </div>
@@ -1381,6 +1435,7 @@
                             <label>Registered Devices</label>
                             <p class="help-block">Register a new device using a QR code, or configure an existing device by clicking the settings icon on the right.</p>
                             <p id="app_api_msg" style="color: #eb8600;">Warning: The API must be enabled under <a data-tab-destination="web_interface" data-target="api_enabled">Web Interface</a> to use the app.</p>
+                            <br />
                             <div class="row">
                                 <div id="plexpy-mobile-devices-table" class="col-md-12">
                                     <div class='text-muted'><i class="fa fa-refresh fa-spin"></i> Loading registered devices...</div>
@@ -1816,7 +1871,10 @@ Rating: {rating}/10   --> Rating: /10
                 <label>Instructions</label>
                 <p class="help-block">
                     Scan the QR code below with the Tautulli Android app to automatically register it with the server (make sure the Tautulli Address below is correct)
-                    or manually enter the connection info and device token into the app settings.
+                    or manually enter the connection info and device token into the app settings. This window will automatically close once device registration is successful.
+                </p>
+                <p class="help-block">
+                    Note: OneSignal.com must not be blocked (e.g. in Pi-hole) for device registration.
                 </p>
                 <label>QR Code</label>
                 <pre id="api_qr_code" style="text-align: center"></pre>
@@ -1844,6 +1902,38 @@ Rating: {rating}/10   --> Rating: /10
     </div>
 </div>
 <div id="mobile-device-config-modal" class="modal fade wide" tabindex="-1" role="dialog" aria-labelledby="mobile-device-config-modal"></div>
+<div id="browse-path-modal" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="browse-path-modal">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-hidden="true"><i class="fa fa-remove"></i></button>
+                <h4 class="modal-title">File Browser</h4>
+            </div>
+            <div class="modal-body">
+                <div class="row">
+                    <div class="col-md-12">
+                        <div class="form-group">
+                            <label for="browse-path">Select a <span id="browse-path-type"></span> Below</label>
+                            <div class="row">
+                                <div class="col-md-12">
+                                    <input type="text" class="form-control" id="browse-path" name="browse-path" value="" size="30" disabled>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                    <div class="col-md-12" style="height: 400px; overflow: auto;">
+                        <ul id="browse-path-list" class="stacked-configs list-unstyled">
+                        </ul>
+                    </div>
+                </div>
+            </div>
+            <div class="modal-footer">
+                <span id="browse-path-status-message" style="padding-right: 25px;"></span>
+                <input type="button" id="select-browse-file" class="btn btn-bright" value="Select">
+            </div>
+        </div>
+    </div>
+</div>
 </%def>
 
 <%def name="javascriptIncludes()">
@@ -1950,6 +2040,38 @@ Rating: {rating}/10   --> Rating: /10
         });
     }
 
+    function browsePath(key, path, filter_ext) {
+        $("#browse-path-status-message").html('<i class="fa fa-fw fa-spin fa-refresh"></i>');
+        getBrowsePath(key, path, filter_ext).then(function (data) {
+            if (data.result === 'error') {
+                $("#browse-path-status-message").html("<i class='fa fa-exclamation-triangle'></i> " + data.message);
+            } else {
+                $("#browse-path-status-message").html("");
+
+                $('#browse-path').val(data.path);
+                var browse_list = $('#browse-path-list');
+                browse_list.parent().animate({ scrollTop: 0 }, 0);
+                browse_list.empty();
+
+                $.each(data.data, function(i, item) {
+                    var browse_item = $('<li/>')
+                        .html("<span><i class='fa fa-fw fa-" + item.icon + "'></i>&nbsp; " + item.title + "</span>")
+                        .addClass(item.type + ' pointer')
+                        .data('key', item.key)
+                        .data('path', item.path)
+                        .appendTo(browse_list)
+                });
+
+                $('#browse-path-list li').click(function (){
+                    $('#browse-path').val($(this).data('path'));
+                    if ($(this).hasClass('folder')) {
+                        browsePath($(this).data('key'), null, filter_ext)
+                    }
+                });
+            }
+        });
+    }
+
 $(document).ready(function() {
 
     // Javascript to enable link to tab
@@ -2036,6 +2158,7 @@ $(document).ready(function() {
     initConfigCheckbox('#https_create_cert');
     initConfigCheckbox('#check_github');
     initConfigCheckbox('#monitor_pms_updates');
+    initConfigCheckbox('#monitor_remote_access');
     initConfigCheckbox('#newsletter_self_hosted');
 
     $('#menu_link_shutdown').click(function() {
@@ -2129,11 +2252,17 @@ $(document).ready(function() {
     });
 
     $("#delete_temp_sessions").click(function () {
-        var msg = 'Are you sure you want to flush the temporary sessions?<br /><strong>This will reset all currently active sessions.</strong>';
+        var msg = 'Are you sure you want to flush the temporary sessions?<br /><br /><strong>This will reset all currently active sessions.</strong>';
         var url = 'delete_temp_sessions';
         confirmAjaxCall(url, msg);
     });
 
+    $("#delete_recently_added").click(function () {
+        var msg = 'Are you sure you want to flush the recently added items?<br /><br /><strong>This will reset all recently added notifications.</strong>';
+        var url = 'delete_recently_added';
+        confirmAjaxCall(url, msg);
+    });
+
     $("#switch_git_branch").click(function () {
         var current_remote = "${config['git_remote']}";
         var current_branch = "${config['git_branch']}";
@@ -2154,6 +2283,17 @@ $(document).ready(function() {
         }
     });
 
+    $("#reset_git_install").click(function () {
+        var msg = 'Are you sure you want to reset your Tautulli installtion back to <strong>${common.RELEASE}</strong>?' +
+                '<br /><br />Tautulli will restart.';
+        $('#confirm-message').html(msg);
+        $('#confirm-modal').modal();
+        $('#confirm-modal').one('click', '#confirm-button', function () {
+            settingsChanged = false;
+            window.location.href = 'reset_git_install';
+        });
+    });
+
     $('#api_key').click(function(){ $('#api_key').select() });
     $("#generate_api").click(function() {
         $.get('generate_api_key',
@@ -2414,9 +2554,7 @@ $(document).ready(function() {
         $("#token_verify").html('<i class="fa fa-refresh fa-spin"></i>').fadeIn('fast');
     }
     function OAuthSuccessCallback(authToken) {
-        var x_plex_headers = getPlexHeaders();
         $("#pms_token").val(authToken);
-        $("#pms_uuid").val(x_plex_headers['X-Plex-Client-Identifier']);
         $("#token_verify").html('<i class="fa fa-check"></i>').fadeIn('fast');
         getServerOptions(authToken);
     }
@@ -2781,12 +2919,23 @@ $(document).ready(function() {
         var name = image_hosting_option.text();
 
         var msg = 'Are you sure you want to delete all uploaded images on <strong>' + name + '</strong>?' +
-            '<br />All previous links to the images will no longer work. This cannot be undone!';
+            '<br /><br />All previous links to the images will no longer work. This cannot be undone!';
         var url = 'delete_hosted_images';
         var data = { service: name, delete_all: true };
         confirmAjaxCall(url, msg, data, false);
     });
 
+    $('body').on('click', '.delete_all_lookups', function () {
+        var service = $(this).data('service');
+        var name = $(this).text();
+
+        var msg = 'Are you sure you want to delete all the metadata lookup info from <strong>' + name + '</strong>?' +
+            '<br /><br />Tautulli will lookup the metadata info again the next time a notification is sent.';
+        var url = 'delete_lookup_info';
+        var data = { service: service, delete_all: true };
+        confirmAjaxCall(url, msg, data, false);
+    });
+
     function baseURLSet() {
         if ($('#http_base_url').val()) {
             $('.base-url-warning').hide();
@@ -2860,54 +3009,46 @@ $(document).ready(function() {
         openPlexXML('/api/resources', true, {includeHttps: 1});
     });
 
-    if ("${kwargs.get('install_geoip')}" === 'true') {
-        gotoSetting('3rd_party_apis', 'geoip_db')
-    }
-
-    if ("${config['geoip_db_installed']}" > "0") {
-        $("#geoip_db_updated").text(moment("${config['geoip_db_installed']}", "X").fromNow());
-    }
-
-    $("#install_geoip_db").click(function () {
-        var maxmind_license_key = $("#maxmind_license_key");
-        maxmind_license_key.val($.trim(maxmind_license_key.val()));
-        if (maxmind_license_key.val() === "") {
-            maxmind_license_key.focus();
-            showMsg('<i class="fa fa-exclamation-circle"></i> Maxmind License Key is required.', false, true, 5000, true);
-            return false;
-        } else if (!(saveSettings())) {
-            return false;
-        }
-        var msg = 'Are you sure you want to install the GeoLite2 database?<br /><br />' +
-            'The database is used to lookup IP address geolocation info.<br />' +
-            'The database will be downloaded from <a href="${anon_url("https://dev.maxmind.com/geoip/geoip2/geolite2/")}" target="_blank">MaxMind</a>, <br />' +
-            'and requires <strong>100MB</strong> of free space to install.<br />';
-        var url = 'install_geoip_db';
-        if ($(this).text() === 'Update') {
-            url += '?update=true';
-        }
-        confirmAjaxCall(url, msg, null, 'Installing GeoLite2 database.', function (result) {
-            if (result.result === "success") {
-                $('#install_geoip_db').text('Update');
-                $('#uninstall_geoip_db').prop('disabled', false);
-                $('#geoip_db_updated').text(moment(result.updated, "X").fromNow());
+    var tautulli_news = $('#tautulli-news')
+    $.ajax({
+        url: 'https://tautulli.com/news/tautulli-news.json',
+        type: 'GET',
+        dataType: 'json',
+        cache: false,
+        async: true,
+        success: function (data) {
+            if (data) {
+                var now = moment().endOf('day');
+                var news = $('<ul/>').addClass('accordion list-unstyled')
+                $.each(data, function (index, news_item) {
+                    var date = moment(news_item.date, "YYYY-MM-DD");
+                    if (index >= 5) { return false; }
+                    var header = $('<div/>').addClass('link').html(
+                        '<span class="toggle-left"><i class="fa fa-newspaper fa-fw"></i></span>' +
+                        '<span class="news-title">' + news_item.title + '</span>' +
+                        '<span class="toggle-right"><i class="fa fa-chevron-down fa-fw"></i></span>' +
+                        '<span class="news-date toggle-right">' + date.format($('#date_format').val()) + '</span>');
+                    var subtitle = $('<span/>').addClass('news-subtitle').html(news_item.subtitle);
+                    var body = $('<span/>').addClass('news-body').html(news_item.body);
+                    var content = $('<div/>').addClass('submenu');
+                    if (news_item.subtitle) { content.append(subtitle); }
+                    content.append(body);
+                    var li = $('<li/>').append(header).append(content)
+                    if (index === 0 && Math.abs(now.diff(date, 'days')) < 7) {
+                        li.addClass('open');
+                        content.css('display', 'block');
+                    }
+                    news.append(li)
+                });
+                tautulli_news.html(news);
+                var accordion_news = new Accordion(news, false);
+            } else {
+                tautulli_news.html('<p class="help-block"><i class="fa fa-check"></i>&nbsp; No news available.</p>')
             }
-            getSchedulerTable();
-        });
-    });
-
-    $("#uninstall_geoip_db").click(function () {
-        var msg = 'Are you sure you want to uninstall the GeoLite2 database?<br /><br />' +
-            'You will not be able to lookup IP address geolocation info.';
-        var url = 'uninstall_geoip_db';
-        confirmAjaxCall(url, msg, null, 'Uninstalling GeoLite2 database.', function (result) {
-            if (result.result === "success") {
-                $('#install_geoip_db').text('Install');
-                $('#uninstall_geoip_db').prop('disabled', true);
-                $('#geoip_db_updated').text('never');
-            }
-            getSchedulerTable();
-        });
+        },
+        error: function () {
+            tautulli_news.html('<p class="help-block"><i class="fa fa-exclamation-triangle"></i>&nbsp; Failed to retrieve news.</p>')
+        }
     });
 });
 </script>

data/interfaces/default/update_metadata.html

@@ -39,30 +39,43 @@ DOCUMENTATION :: END
                         <ul class="list-unstyled breadcrumb">
                             % if query['media_type'] == 'movie':
                             <li>Movies</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="active">${query['title']}</li>
                             % elif query['media_type'] == 'show':
                             <li>TV Shows</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="active">${query['grandparent_title']}</li>
                             % elif query['media_type'] == 'season':
                             <li class="hidden-xs hidden-sm">TV Shows</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="hidden-xs hidden-sm">${query['grandparent_title']}</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="active">Season ${query['parent_media_index']}</li>
                             % elif query['media_type'] == 'episode':
                             <li class="hidden-xs hidden-sm">TV Shows</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="hidden-xs hidden-sm">${query['grandparent_title']}</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li>Season ${query['parent_media_index']}</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="active">Episode ${query['media_index']} - ${query['title']}</li>
                             % elif query['media_type'] == 'artist':
-                            <li><Music</li>
+                            <li>Music</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="active">${query['grandparent_title']}</li>
                             % elif query['media_type'] == 'album':
                             <li class="hidden-xs hidden-sm">Music</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li>${query['grandparent_title']}</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="active">${query['parent_title']}</li>
                             % elif query['media_type'] == 'track':
                             <li class="hidden-xs hidden-sm">Music</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="hidden-xs hidden-sm">${query['grandparent_title']}</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li>${query['parent_title']}</li>
+                            <span class="breadcrumb-arrow"><i class="fa fa-chevron-right"></i></span>
                             <li class="active">Track ${query['media_index']} - ${query['title']}</li>
                             % endif
                         </ul>

data/interfaces/default/user.html

@@ -51,7 +51,13 @@ DOCUMENTATION :: END
                 <div class="col-md-12">
                     <div class="table-card-back">
                         <div class="user-info-wrapper">
-                            <div class="user-info-poster-face" style="background-image: url(${data['user_thumb']});"></div>
+                            <div class="user-info-poster-face" style="background-image: url(${data['user_thumb']});">
+                                % if not data['is_active']:
+                                <span class="inactive-user-tooltip" data-toggle="tooltip" title="User not on Plex server">
+                                    <i class="fa fa-2x fa-exclamation-triangle"></i>
+                                </span>
+                                % endif
+                            </div>
                             <div class="user-info-username">
                                 <span class="set-username">${data['friendly_name']}</span>
                                 % if _session['user_group'] == 'admin':
@@ -540,6 +546,8 @@ DOCUMENTATION :: END
         login_log_table.draw();
     });
 
+    $(".inactive-user-tooltip").tooltip();
+
     % if _session['user_group'] == 'admin':
     $("#edit-user-tooltip").tooltip();
 
@@ -566,19 +574,17 @@ DOCUMENTATION :: END
                 $('#deleteType').text('history');
                 $('#confirm-modal-delete').modal();
                 $('#confirm-modal-delete').one('click', '#confirm-delete', function () {
-                    history_to_delete.forEach(function(row, idx) {
-                        $.ajax({
-                            url: 'delete_history_rows',
-                            type: 'POST',
-                            data: { row_id: row },
-                            async: true,
-                            success: function (data) {
-                                var msg = "History deleted";
-                                showMsg(msg, false, true, 2000);
-                            }
-                        });
+                    $.ajax({
+                        url: 'delete_history_rows',
+                        type: 'POST',
+                        data: { row_ids: history_to_delete.join(',') },
+                        async: true,
+                        success: function (data) {
+                            var msg = "History deleted";
+                            showMsg(msg, false, true, 2000);
+                            history_table.draw();
+                        }
                     });
-                    history_table.draw();
                 });
             }
 

data/interfaces/default/user_player_stats.html

@@ -27,7 +27,7 @@ DOCUMENTATION :: END
             <div id="user-player-image-${a['result_id']}">
                 <div class="user-player-instance-box svg-icon platform-${a['platform_name']}"></div>
             </div>
-            <div class="user-player-instance-name">
+            <div class="user-player-instance-name" title="${a['player_name']}">
                 ${a['player_name']}
             </div>
             <div class="user-player-instance-playcount">

data/interfaces/default/users.html

@@ -119,14 +119,14 @@
                     });
 
                     if (users_to_delete.length > 0) {
-                        $('#users-to-delete').prepend('<p>Are you REALLY sure you want to delete and purge all history for the following users:</p>')
+                        $('#users-to-delete').prepend('<p>Are you REALLY sure you want to delete and purge all history for the following users:</p>');
                         for (var i = 0; i < users_to_delete.length; i++) {
                             $('#users-to-delete').append('<li>' + $('div[data-id=' + users_to_delete[i] + '] > input').val() + '</li>');
                         }
                     }
 
                     if (users_to_purge.length > 0) {
-                        $('#users-to-purge').prepend('<p>Are you REALLY sure you want to purge all history for the following users:</p>')
+                        $('#users-to-purge').prepend('<p>Are you REALLY sure you want to purge all history for the following users:</p>');
                         for (var i = 0; i < users_to_purge.length; i++) {
                             $('#users-to-purge').append('<li>' + $('div[data-id=' + users_to_purge[i] + '] > input').val() + '</li>');
                         }
@@ -134,33 +134,30 @@
 
                     $('#confirm-modal-delete').modal();
                     $('#confirm-modal-delete').one('click', '#confirm-delete', function () {
-                        users_to_delete.forEach(function(row, idx) {
-                            $.ajax({
-                                url: 'delete_user',
-                                type: 'POST',
-                                data: { user_id: row },
-                                cache: false,
-                                async: true,
-                                success: function (data) {
-                                    var msg = "User deleted";
-                                    showMsg(msg, false, true, 2000);
-                                }
-                            });
+                        $.ajax({
+                            url: 'delete_all_user_history',
+                            type: 'POST',
+                            data: { row_ids: users_to_purge.join(',') },
+                            cache: false,
+                            async: true,
+                            success: function (data) {
+                                var msg = "User history purged";
+                                showMsg(msg, false, true, 2000);
+                                users_list_table.draw();
+                            }
                         });
-                        users_to_purge.forEach(function(row, idx) {
-                            $.ajax({
-                                url: 'delete_all_user_history',
-                                type: 'POST',
-                                data: { user_id: row },
-                                cache: false,
-                                async: true,
-                                success: function (data) {
-                                    var msg = "User history purged";
-                                    showMsg(msg, false, true, 2000);
-                                }
-                            });
+                        $.ajax({
+                            url: 'delete_user',
+                            type: 'POST',
+                            data: { row_ids: users_to_delete.join(',') },
+                            cache: false,
+                            async: true,
+                            success: function (data) {
+                                var msg = "User deleted";
+                                showMsg(msg, false, true, 2000);
+                                users_list_table.draw();
+                            }
                         });
-                        users_list_table.draw();
                     });
                 }
 
@@ -192,7 +189,7 @@
             complete: function (xhr, status) {
                 var result = $.parseJSON(xhr.responseText);
                 var msg = result.message;
-                if (result.result == 'success') {
+                if (result.result === 'success') {
                     showMsg('<i class="fa fa-check"></i> ' + msg, false, true, 2000, false);
                     users_list_table.draw();
                 } else {

data/interfaces/default/welcome.html

@@ -203,7 +203,7 @@
                     <h3>Database Import</h3>
                     <div class="wizard-input-section">
                         <p class="help-block">
-                            If you have an existing PlexWatch/Plexivity database, you can import the data into Tautulli.
+                            If you have an existing Tautulli, PlexWatch, or Plexivity database, you can import the data into Tautulli.
                         </p>
                         <p class="help-block">
                             To import a database, navigate to the <strong>Settings</strong> page
@@ -216,6 +216,8 @@
                         <input type="checkbox" name="first_run" id="first_run" value="1" checked>
                         <input type="checkbox" name="group_history_tables" id="group_history_tables" value="1" checked>
                         <input type="checkbox" name="history_table_activity" id="history_table_activity" value="1" checked>
+                        <input type="checkbox" name="sys_tray_icon" id="sys_tray_icon" value="1" checked>
+                        <input type="checkbox" name="launch_startup" id="launch_startup" value="1" checked>
                         <input type="checkbox" name="launch_browser" id="launch_browser" value="1" checked>
                         <input type="checkbox" name="api_enabled" id="api_enabled" value="1" checked>
                         <input type="checkbox" name="refresh_users_on_startup" id="refresh_users_on_startup" value="1" checked>
@@ -223,6 +225,8 @@
                         <input type="checkbox" name="check_github" id="check_github" value="1" checked>
                         <input type="checkbox" name="log_blacklist" id="log_blacklist" value="1" checked>
                         <input type="checkbox" name="cache_images" id="cache_images" value="1" checked>
+                        <input type="checkbox" name="notify_group_recently_added_grandparent" id="notify_group_recently_added_grandparent" value="1" checked>
+                        <input type="checkbox" name="notify_group_recently_added_parent" id="notify_group_recently_added_parent" value="1" checked>
                         <input type="checkbox" name="server_changed" id="server_changed" value="1" checked>
                         <input type="checkbox" name="first_run_complete" id="first_run_complete" value="1" checked>
                         <input type="text" name="home_stats_cards" id="home_stats_cards" value="first_run_wizard">
@@ -491,7 +495,7 @@ $(document).ready(function() {
             var pms_ssl = $("#pms_ssl").val();
             var pms_is_remote = $("#pms_is_remote").val();
             if ((pms_ip !== '') || (pms_port !== '')) {
-                $("#pms-verify-status").html('<i class="fa fa-refresh fa-spin"></i>&nbsp; Validating server...');
+                $("#pms-verify-status").html('<i class="fa fa-refresh fa-spin"></i>&nbsp; Verifying server...');
                 $('#pms-verify-status').fadeIn('fast');
                 $.ajax({
                     url: 'get_server_id',
@@ -506,7 +510,7 @@ $(document).ready(function() {
                     async: true,
                     timeout: 5000,
                     error: function (jqXHR, textStatus, errorThrown) {
-                        $("#pms-verify-status").html('<i class="fa fa-exclamation-circle"></i>&nbsp; This is not a Plex Server!');
+                        $("#pms-verify-status").html('<i class="fa fa-exclamation-circle"></i>&nbsp; Error verifying server: ' + textStatus);
                         $('#pms-verify-status').fadeIn('fast');
                     },
                     success: function(xhr, status) {

data/interfaces/newsletters/recently_added.html

@@ -26,6 +26,7 @@
     <meta name="viewport" content="width=device-width">
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <title>Tautulli Newsletter - ${subject}</title>
+    <link rel="shortcut icon" href="${base_url_image + 'images/favicon/favicon.ico' if base_url_image else 'https://tautulli.com/images/favicon.ico'}">
     <style>
         /* -------------------------------------
             GLOBAL RESETS

data/interfaces/newsletters/recently_added.internal.html

@@ -26,6 +26,7 @@
     <meta name="viewport" content="width=device-width"/>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
     <title>Tautulli Newsletter - ${subject}</title>
+    <link rel="shortcut icon" href="${base_url_image + 'images/favicon/favicon.ico' if base_url_image else 'https://tautulli.com/images/favicon.ico'}">
     <style>
         /* -------------------------------------
             GLOBAL RESETS

init-scripts/init-alt.freebsd

@@ -1,88 +0,0 @@
-#!/bin/sh
-#
-# PROVIDE: tautulli
-# REQUIRE: tautulli
-# KEYWORD: shutdown
-#
-# Add the following lines to /etc/rc.conf.local or /etc/rc.conf
-# to enable this service:
-#
-# tautulli_enable (bool):     Set to NO by default.
-#                       Set it to YES to enable it.
-# tautulli_user:  The user account Tautulli daemon runs as what
-#                       you want it to be. It uses 'tautulli' user by
-#                       default. Do not sets it as empty or it will run
-#                       as root.
-# tautulli_dir:       Directory where Tautulli lives.
-#                       Default: /usr/local/share/Tautulli
-# tautulli_chdir:  Change to this directory before running Tautulli.
-#     Default is same as tautulli_dir.
-# tautulli_pid:  The name of the pidfile to create.
-#     Default is tautulli.pid in tautulli_dir.
-
-. /etc/rc.subr
-
-name="tautulli"
-rcvar=${name}_enable
-
-load_rc_config ${name}
-
-: ${tautulli_enable:="NO"}
-: ${tautulli_user:="tautulli"}
-: ${tautulli_dir:="/usr/local/share/Tautulli"}
-: ${tautulli_chdir:="${tautulli_dir}"}
-: ${tautulli_pid:="${tautulli_dir}/tautulli.pid"}
-: ${tautulli_conf:="${tautulli_dir}/config.ini"}
-
-WGET="/usr/local/bin/wget" # You need wget for this script to safely shutdown Tautulli.
-if [ -e "${tautulli_conf}" ]; then
-        HOST=`grep -A64 "\[General\]" "${tautulli_conf}"|egrep "^http_host"|perl -wple 's/^http_host = (.*)$/$1/'`
-        PORT=`grep -A64 "\[General\]" "${tautulli_conf}"|egrep "^http_port"|perl -wple 's/^http_port = (.*)$/$1/'`
-fi
-
-status_cmd="${name}_status"
-stop_cmd="${name}_stop"
-
-command="${tautulli_dir}/Tautulli.py"
-command_args="--daemon --quiet --nolaunch --port ${PORT} --pidfile ${tautulli_pid} --config ${tautulli_conf}"
-
-# Check for wget and refuse to start without it.
-if [ ! -x "${WGET}" ]; then
-  warn "Tautulli not started: You need wget to safely shut down Tautulli."
-  exit 1
-fi
-
-# Ensure user is root when running this script.
-if [ `id -u` != "0" ]; then
-  echo "Oops, you should be root before running this!"
-  exit 1
-fi
-
-verify_tautulli_pid() {
-    # Make sure the pid corresponds to the Tautulli process.
-    pid=`cat ${tautulli_pid} 2>/dev/null`
-    ps -p ${pid} | grep -q "python ${tautulli_dir}/Tautulli.py"
-    return $?
-}
-
-# Try to stop Tautulli cleanly by calling shutdown over http.
-tautulli_stop() {
-    if [ ! -e "${tautulli_conf}" ]; then
-        echo "Tautulli' settings file does not exist. Try starting Tautulli, as this should create the file."
-        exit 1
-    fi
-    echo "Stopping $name"
-    verify_tautulli_pid
-    ${WGET} -O - -q --user=${SBUSR} --password=${SBPWD} "http://${HOST}:${PORT}/shutdown/" >/dev/null
-    
-    if [ -n "${pid}" ]; then
-      wait_for_pids ${pid}
-      echo "Stopped $name"
-    fi
-}
-
-tautulli_status() {
-    verify_tautulli_pid && echo "$name is running as ${pid}" || echo "$name is not running"
-}
-
-run_rc_command "$1"

init-scripts/init.fedora.centos.service

@@ -1,76 +0,0 @@
-#!/bin/sh
-#
-### BEGIN INIT INFO
-# Provides:          Tautulli
-# Required-Start:    $all
-# Required-Stop:     $all
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: starts Tautulli
-# Description:       starts Tautulli
-### END INIT INFO
-
-# Source function library.
-. /etc/init.d/functions
-
-## Variables
-prog=tautulli
-lockfile=/var/lock/subsys/$prog
-homedir=/opt/Tautulli
-datadir=/opt/Tautulli
-configfile=/opt/Tautulli/config.ini
-pidfile=/var/run/tautulli.pid
-nice=
-# The following line must point to your Python 2.7 install
-python27=/usr/src/Python-2.7.11/python
-##
-
-options=" --daemon --config $configfile --pidfile $pidfile --datadir $datadir --nolaunch --quiet"
-
-start() {
-        # Start daemon.
-        echo -n $"Starting $prog: "
-        daemon --pidfile=$pidfile $nice $python27 $homedir/Tautulli.py $options
-        RETVAL=$?
-        echo
-        [ $RETVAL -eq 0 ] && touch $lockfile
-        return $RETVAL
-}
-
-stop() {
-        echo -n $"Shutting down $prog: "
-        killproc -p $pidfile $python27
-        RETVAL=$?
-        echo
-        [ $RETVAL -eq 0 ] && rm -f $lockfile
-        return $RETVAL
-}
-
-# See how we were called.
-case "$1" in
-  start)
-        start
-        ;;
-  stop)
-        stop
-        ;;
-  status)
-        status $prog
-        ;;
-  restart|force-reload)
-        stop
-        start
-        ;;
-  try-restart|condrestart)
-        if status $prog > /dev/null; then
-            stop
-            start
-        fi
-        ;;
-  reload)
-        exit 3
-        ;;
-  *)
-        echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}"
-        exit 2
-esac

init-scripts/init.freebsd

@@ -51,7 +51,7 @@ verify_tautulli_pid() {
     # Make sure the pid corresponds to the Tautulli process.
     if [ -f ${tautulli_pid} ]; then
       pid=`cat ${tautulli_pid} 2>/dev/null`
-      ps -p ${pid} | grep -q "python2 ${tautulli_dir}/Tautulli.py"
+      ps -p ${pid} | grep -q "python ${tautulli_dir}/Tautulli.py"
       return $?
     else
       return 0
@@ -60,7 +60,7 @@ verify_tautulli_pid() {
 
 # Try to stop Tautulli cleanly by sending SIGTERM
 tautulli_stop() {
-    echo "Stopping $name"
+    echo "Stopping $name."
     verify_tautulli_pid
     if [ -n "${pid}" ]; then
       kill ${pid}

init-scripts/init.freenas

@@ -1,81 +0,0 @@
-#!/bin/sh
-#
-# PROVIDE: tautulli
-# REQUIRE: DAEMON tautulli
-# KEYWORD: shutdown
-#
-# Add the following lines to /etc/rc.conf.local or /etc/rc.conf
-# to enable this service:
-#
-# tautulli_enable (bool): Set to NO by default.
-#           Set it to YES to enable it.
-# tautulli_user:  The user account Tautulli daemon runs as what
-#           you want it to be. It uses 'tautulli' user by
-#           default. Do not sets it as empty or it will run
-#           as root.
-# tautulli_dir:   Directory where Tautulli lives.
-#           Default: /usr/local/share/Tautulli
-# tautulli_chdir:  Change to this directory before running Tautulli.
-#     Default is same as tautulli_dir.
-# tautulli_pid:  The name of the pidfile to create.
-#     Default is tautulli.pid in tautulli_dir.
-PATH="/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
-
-. /etc/rc.subr
-
-name="tautulli"
-rcvar=${name}_enable
-
-load_rc_config ${name}
-
-: ${tautulli_enable:="NO"}
-: ${tautulli_user:="tautulli"}
-: ${tautulli_dir:="/usr/local/share/Tautulli"}
-: ${tautulli_chdir:="${tautulli_dir}"}
-: ${tautulli_pid:="${tautulli_dir}/tautulli.pid"}
-: ${tautulli_flags:=""}
-
-status_cmd="${name}_status"
-stop_cmd="${name}_stop"
-
-command="${tautulli_dir}/Tautulli.py"
-command_args="--daemon --pidfile ${tautulli_pid} --quiet --nolaunch ${tautulli_flags}"
-
-# Ensure user is root when running this script.
-if [ `id -u` != "0" ]; then
-  echo "Oops, you should be root before running this!"
-  exit 1
-fi
-
-verify_tautulli_pid() {
-    # Make sure the pid corresponds to the Tautulli process.
-    if [ -f ${tautulli_pid} ]; then
-      pid=`cat ${tautulli_pid} 2>/dev/null`
-      ps -p ${pid} | grep -q "python2 ${tautulli_dir}/Tautulli.py"
-      return $?
-    else
-      return 0
-    fi
-}
-
-# Try to stop Tautulli cleanly by sending SIGTERM
-tautulli_stop() {
-    echo "Stopping $name."
-    verify_tautulli_pid
-    if [ -n "${pid}" ]; then
-      kill ${pid}
-      wait_for_pids ${pid}
-      echo "Stopped."
-    fi
-}
-
-tautulli_status() {
-    verify_tautulli_pid
-    if [ -n "${pid}" ]; then
-      echo "$name is running as ${pid}."
-    else
-      echo "$name is not running."
-    fi
-}
-
-run_rc_command "$1"

init-scripts/init.freenas

@@ -0,0 +1 @@
+init.freebsd

init-scripts/init.solaris11

@@ -1,47 +0,0 @@
-<?xml version="1.0"?>
-<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
-<!--
-        Created by Manifold
---><service_bundle type="manifest" name="tautulli">
-
-    <service name="application/tautulli" type="service" version="1">
-
-        <create_default_instance enabled="true"/>
-
-        <single_instance/>
-
-        <dependency name="network" grouping="require_all" restart_on="error" type="service">
-            <service_fmri value="svc:/milestone/network:default"/>
-        </dependency>
-
-        <dependency name="filesystem" grouping="require_all" restart_on="error" type="service">
-            <service_fmri value="svc:/system/filesystem/local"/>
-        </dependency>
-
-        <method_context>
-            <method_credential user="tautulli" group="nogroup"/>
-        </method_context>
-
-        <exec_method type="method" name="start" exec="python /opt/Tautulli/Tautulli.py --daemon --quiet --nolaunch" timeout_seconds="60"/>
-
-        <exec_method type="method" name="stop" exec=":kill" timeout_seconds="60"/>
-
-        <property_group name="startd" type="framework">
-            <propval name="duration" type="astring" value="contract"/>
-            <propval name="ignore_error" type="astring" value="core,signal"/>
-        </property_group>
-
-
-        <stability value="Evolving"/>
-
-        <template>
-            <common_name>
-                <loctext xml:lang="C">
-                    Tautulli
-                </loctext>
-            </common_name>
-        </template>
-
-    </service>
-
-</service_bundle>

init-scripts/init.ubuntu

@@ -1,209 +0,0 @@
-#!/bin/sh
-#
-## Don't edit this file
-## Edit user configuation in /etc/default/tautulli to change
-##
-## Make sure init script is executable
-## sudo chmod +x /path/to/init.ubuntu
-##
-## Install the init script
-## sudo ln -s /path/to/init.ubuntu /etc/init.d/tautulli
-##
-## Create the tautulli daemon user:
-## sudo adduser --system --no-create-home tautulli
-##
-## Make sure /opt/Tautulli is owned by the tautulli user
-## sudo chown tautulli:nogroup -R /opt/Tautulli
-##
-## Touch the default file to stop the warning message when starting
-## sudo touch /etc/default/tautulli
-##
-## To start Tautulli automatically
-## sudo  update-rc.d tautulli defaults
-##
-## To start/stop/restart Tautulli
-## sudo service tautulli start
-## sudo service tautulli stop
-## sudo service tautulli restart
-##
-## TAUTULLI_USER=         #$RUN_AS, username to run Tautulli under, the default is tautulli
-## TAUTULLI_HOME=         #$APP_PATH, the location of Tautulli.py, the default is /opt/Tautulli
-## TAUTULLI_DATA=         #$DATA_DIR, the location of plexpy.db, cache, logs, the default is /opt/Tautulli
-## TAUTULLI_PIDFILE=      #$PID_FILE, the location of tautulli.pid, the default is /var/run/tautulli/tautulli.pid
-## PYTHON_BIN=      #$DAEMON, the location of the python binary, the default is /usr/bin/python
-## TAUTULLI_OPTS=         #$EXTRA_DAEMON_OPTS, extra cli option for Tautulli, i.e. " --config=/home/Tautulli/config.ini"
-## SSD_OPTS=        #$EXTRA_SSD_OPTS, extra start-stop-daemon option like " --group=users"
-## TAUTULLI_PORT=         #$PORT_OPTS, hardcoded port for the webserver, overrides value in config.ini
-##
-## EXAMPLE if want to run as different user
-## add TAUTULLI_USER=username to /etc/default/tautulli
-## otherwise default tautulli is used
-#
-### BEGIN INIT INFO
-# Provides:          tautulli
-# Required-Start:    $local_fs $network $remote_fs
-# Required-Stop:     $local_fs $network $remote_fs
-# Should-Start:      $NetworkManager
-# Should-Stop:       $NetworkManager
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: starts instance of Tautulli
-# Description:       starts instance of Tautulli using start-stop-daemon
-### END INIT INFO
-
-# Script name
-NAME=tautulli
-
-# App name
-DESC=Tautulli
-
-SETTINGS_LOADED=FALSE
-
-. /lib/lsb/init-functions
-
-# Source Tautulli configuration
-if [ -f /etc/default/tautulli ]; then
-    SETTINGS=/etc/default/tautulli
-else
-    log_warning_msg "/etc/default/tautulli not found using default settings.";
-fi
-
-check_retval() {
-    if [ $? -eq 0 ]; then
-        log_end_msg 0
-        return 0
-    else
-        log_end_msg 1
-        exit 1
-    fi
-}
-
-load_settings() {
-    if [ $SETTINGS_LOADED != "TRUE" ]; then
-        . $SETTINGS
-
-        ## The defaults
-        # Run as username
-        RUN_AS=${TAUTULLI_USER-tautulli}
-
-        # Path to app TAUTULLI_HOME=path_to_app_Tautulli.py
-        APP_PATH=${TAUTULLI_HOME-/opt/Tautulli}
-
-        # Data directory where plexpy.db, cache and logs are stored
-        DATA_DIR=${TAUTULLI_DATA-/opt/Tautulli}
-
-        # Path to store PID file
-        PID_FILE=${TAUTULLI_PIDFILE-/var/run/tautulli/tautulli.pid}
-
-        # Path to python bin
-        DAEMON=${PYTHON_BIN-/usr/bin/python}
-
-        # Extra daemon option like: TAUTULLI_OPTS=" --config=/home/Tautulli/config.ini"
-        EXTRA_DAEMON_OPTS=${TAUTULLI_OPTS-}
-
-        # Extra start-stop-daemon option like START_OPTS=" --group=users"
-        EXTRA_SSD_OPTS=${SSD_OPTS-}
-
-        # Hardcoded port to run on, overrides config.ini settings
-        [ -n "$TAUTULLI_PORT" ] && {
-            PORT_OPTS=" --port=${TAUTULLI_PORT} "
-        }
-
-        DAEMON_OPTS=" Tautulli.py --quiet --daemon --nolaunch --pidfile=${PID_FILE} --datadir=${DATA_DIR} ${PORT_OPTS}${EXTRA_DAEMON_OPTS}"
-
-        SETTINGS_LOADED=TRUE
-    fi
-
-    [ -x $DAEMON ] || {
-        log_warning_msg "$DESC: Can't execute daemon, aborting. See $DAEMON";
-        return 1;}
-
-    return 0
-}
-
-load_settings || exit 0
-
-is_running () {
-    # returns 1 when running, else 0.
-    if [ -e $PID_FILE ]; then
-      PID=`cat $PID_FILE`
-
-      RET=$?
-      [ $RET -gt 1 ] && exit 1 || return $RET
-    else
-      return 1
-    fi
-}
-
-handle_pid () {
-    PID_PATH=`dirname $PID_FILE`
-    [ -d $PID_PATH ] || mkdir -p $PID_PATH && chown -R $RUN_AS $PID_PATH > /dev/null || {
-        log_warning_msg "$DESC: Could not create $PID_FILE, See $SETTINGS, aborting.";
-        return 1;}
-
-    if [ -e $PID_FILE ]; then
-        PID=`cat $PID_FILE`
-        if ! kill -0 $PID > /dev/null 2>&1; then
-            log_warning_msg "Removing stale $PID_FILE"
-            rm $PID_FILE
-        fi
-    fi
-}
-
-handle_datadir () {
-    [ -d $DATA_DIR ] || mkdir -p $DATA_DIR && chown -R $RUN_AS $DATA_DIR > /dev/null || {
-        log_warning_msg "$DESC: Could not create $DATA_DIR, See $SETTINGS, aborting.";
-        return 1;}
-}
-
-handle_updates () {
-    chown -R $RUN_AS $APP_PATH > /dev/null || {
-        log_warning_msg "$DESC: $APP_PATH not writable by $RUN_AS for web-updates";
-        return 0; }
-}
-
-start_tautulli () {
-    handle_pid
-    handle_datadir
-    handle_updates
-    if ! is_running; then
-        log_daemon_msg "Starting $DESC"
-        start-stop-daemon -o -d $APP_PATH -c $RUN_AS --start $EXTRA_SSD_OPTS --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS
-        check_retval
-    else
-        log_success_msg "$DESC: already running (pid $PID)"
-    fi
-}
-
-stop_tautulli () {
-    if is_running; then
-        log_daemon_msg "Stopping $DESC"
-        start-stop-daemon -o --stop --pidfile $PID_FILE --retry 15
-        check_retval
-    else
-        log_success_msg "$DESC: not running"
-    fi
-}
-
-case "$1" in
-    start)
-        start_tautulli
-        ;;
-    stop)
-        stop_tautulli
-        ;;
-    restart|force-reload)
-        stop_tautulli
-        start_tautulli
-        ;;
-    status)
-        status_of_proc -p "$PID_FILE" "$DAEMON" "$DESC"
-        ;;
-    *)
-        N=/etc/init.d/$NAME
-        echo "Usage: $N {start|stop|restart|force-reload|status}" >&2
-        exit 1
-        ;;
-esac
-
-exit 0

init-scripts/init.upstart

@@ -1,18 +0,0 @@
-# tautulli
-#
-# This is a session/user job. Install this file into /usr/share/upstart/sessions
-# if Tautulli is installed system wide, and into $XDG_CONFIG_HOME/upstart if
-# Tautulli is installed per user. Change the executable path appropiately.
-
-start on desktop-start
-stop on desktop-end
-
-env CONFIG=""$XDG_CONFIG_HOME"/Tautulli"
-env DATA=""$XDG_DATA_HOME"/Tautulli"
-
-pre-start script
-    [ -d "$CONFIG" ] || mkdir -p "$CONFIG"
-    [ -d "$DATA" ] || mkdir -p "$DATA"
-end script
-
-exec Tautulli.py --nolaunch --config "$CONFIG"/config.ini --datadir "$DATA"

lib/UniversalAnalytics/HTTPLog.py

@@ -10,7 +10,7 @@
 
 
 import sys, re, os
-from cStringIO import StringIO
+from io import StringIO
 
 
 
@@ -116,6 +116,6 @@ def consume(outbuffer = None): # Capture standard output
 
 if __name__ == '__main__':
     consume(sys.stdout).write(sys.stdin.read())
-    print '\n'
+    print('\n')
 
 # vim: set nowrap tabstop=4 shiftwidth=4 softtabstop=0 expandtab textwidth=0 filetype=python foldmethod=indent foldcolumn=4

lib/UniversalAnalytics/Tracker.py

@@ -1,16 +1,7 @@
-###############################################################################
-# Universal Analytics for Python
-# Copyright (c) 2013, Analytics Pros
-# 
-# This project is free software, distributed under the BSD license. 
-# Analytics Pros offers consulting and integration services if your firm needs 
-# assistance in strategy, implementation, or auditing existing work.
-###############################################################################
-
-from urllib2 import urlopen, build_opener, install_opener
-from urllib2 import Request, HTTPSHandler
-from urllib2 import URLError, HTTPError
-from urllib import urlencode
+from future.moves.urllib.request import urlopen, build_opener, install_opener
+from future.moves.urllib.request import Request, HTTPSHandler
+from future.moves.urllib.error import URLError, HTTPError
+from future.moves.urllib.parse import urlencode
 
 import random
 import datetime
@@ -24,8 +15,8 @@ def generate_uuid(basedata=None):
     """ Provides a _random_ UUID with no input, or a UUID4-format MD5 checksum of any input data provided """
     if basedata is None:
         return str(uuid.uuid4())
-    elif isinstance(basedata, basestring):
-        checksum = hashlib.md5(basedata).hexdigest()
+    elif isinstance(basedata, str):
+        checksum = hashlib.md5(str(basedata).encode('utf-8')).hexdigest()
         return '%8s-%4s-%4s-%4s-%12s' % (
         checksum[0:8], checksum[8:12], checksum[12:16], checksum[16:20], checksum[20:32])
 
@@ -44,7 +35,7 @@ class Time(datetime.datetime):
     def to_unix(cls, timestamp):
         """ Wrapper over time module to produce Unix epoch time as a float """
         if not isinstance(timestamp, datetime.datetime):
-            raise TypeError, 'Time.milliseconds expects a datetime object'
+            raise TypeError('Time.milliseconds expects a datetime object')
         base = time.mktime(timestamp.timetuple())
         return base
 
@@ -86,14 +77,14 @@ class HTTPRequest(object):
     def fixUTF8(cls, data):  # Ensure proper encoding for UA's servers...
         """ Convert all strings to UTF-8 """
         for key in data:
-            if isinstance(data[key], basestring):
+            if isinstance(data[key], str):
                 data[key] = data[key].encode('utf-8')
         return data
 
     # Apply stored properties to the given dataset & POST to the configured endpoint
     def send(self, data):
         request = Request(
-            self.endpoint + '?' + urlencode(self.fixUTF8(data)),
+            self.endpoint + '?' + urlencode(self.fixUTF8(data)).encode('utf-8'),
             headers={
                 'User-Agent': self.user_agent
             }
@@ -121,7 +112,7 @@ class HTTPPost(HTTPRequest):
     def send(self, data):
         request = Request(
             self.endpoint,
-            data=urlencode(self.fixUTF8(data)),
+            data=urlencode(self.fixUTF8(data)).encode('utf-8'),
             headers={
                 'User-Agent': self.user_agent
             }
@@ -144,26 +135,26 @@ class Tracker(object):
 
     @classmethod
     def coerceParameter(cls, name, value=None):
-        if isinstance(name, basestring) and name[0] == '&':
+        if isinstance(name, str) and name[0] == '&':
             return name[1:], str(value)
         elif name in cls.parameter_alias:
             typecast, param_name = cls.parameter_alias.get(name)
             return param_name, typecast(value)
         else:
-            raise KeyError, 'Parameter "{0}" is not recognized'.format(name)
+            raise KeyError('Parameter "{0}" is not recognized'.format(name))
 
     def payload(self, data):
-        for key, value in data.iteritems():
+        for key, value in data.items():
             try:
                 yield self.coerceParameter(key, value)
             except KeyError:
                 continue
 
     option_sequence = {
-        'pageview': [(basestring, 'dp')],
-        'event': [(basestring, 'ec'), (basestring, 'ea'), (basestring, 'el'), (int, 'ev')],
-        'social': [(basestring, 'sn'), (basestring, 'sa'), (basestring, 'st')],
-        'timing': [(basestring, 'utc'), (basestring, 'utv'), (basestring, 'utt'), (basestring, 'utl')]
+        'pageview': [(str, 'dp')],
+        'event': [(str, 'ec'), (str, 'ea'), (str, 'el'), (int, 'ev')],
+        'social': [(str, 'sn'), (str, 'sa'), (str, 'st')],
+        'timing': [(str, 'utc'), (str, 'utv'), (str, 'utt'), (str, 'utl')]
     }
 
     @classmethod
@@ -232,7 +223,7 @@ class Tracker(object):
                 for key, val in self.payload(item):
                     data[key] = val
 
-        for k, v in self.params.iteritems():  # update only absent parameters
+        for k, v in self.params.items():  # update only absent parameters
             if k not in data:
                 data[k] = v
 
@@ -247,13 +238,13 @@ class Tracker(object):
     # Setting persistent attibutes of the session/hit/etc (inc. custom dimensions/metrics)
     def set(self, name, value=None):
         if isinstance(name, dict):
-            for key, value in name.iteritems():
+            for key, value in name.items():
                 try:
                     param, value = self.coerceParameter(key, value)
                     self.params[param] = value
                 except KeyError:
                     pass
-        elif isinstance(name, basestring):
+        elif isinstance(name, str):
             try:
                 param, value = self.coerceParameter(name, value)
                 self.params[param] = value
@@ -277,7 +268,7 @@ class Tracker(object):
 def safe_unicode(obj):
     """ Safe convertion to the Unicode string version of the object """
     try:
-        return unicode(obj)
+        return str(obj)
     except UnicodeDecodeError:
         return obj.decode('utf-8')
 
@@ -380,7 +371,7 @@ for i in range(0, 5):
 # Enhanced Ecommerce
 Tracker.alias(str, 'pa')  # Product action
 Tracker.alias(str, 'tcc')  # Coupon code
-Tracker.alias(unicode, 'pal')  # Product action list
+Tracker.alias(str, 'pal')  # Product action list
 Tracker.alias(int, 'cos')  # Checkout step
 Tracker.alias(str, 'col')  # Checkout step option
 
@@ -388,10 +379,10 @@ Tracker.alias(str, 'promoa')  # Promotion action
 
 for product_index in range(1, MAX_EC_PRODUCTS):
     Tracker.alias(str, 'pr{0}id'.format(product_index))  # Product SKU
-    Tracker.alias(unicode, 'pr{0}nm'.format(product_index))  # Product name
-    Tracker.alias(unicode, 'pr{0}br'.format(product_index))  # Product brand
-    Tracker.alias(unicode, 'pr{0}ca'.format(product_index))  # Product category
-    Tracker.alias(unicode, 'pr{0}va'.format(product_index))  # Product variant
+    Tracker.alias(str, 'pr{0}nm'.format(product_index))  # Product name
+    Tracker.alias(str, 'pr{0}br'.format(product_index))  # Product brand
+    Tracker.alias(str, 'pr{0}ca'.format(product_index))  # Product category
+    Tracker.alias(str, 'pr{0}va'.format(product_index))  # Product variant
     Tracker.alias(str, 'pr{0}pr'.format(product_index))  # Product price
     Tracker.alias(int, 'pr{0}qt'.format(product_index))  # Product quantity
     Tracker.alias(str, 'pr{0}cc'.format(product_index))  # Product coupon code
@@ -403,10 +394,10 @@ for product_index in range(1, MAX_EC_PRODUCTS):
 
     for list_index in range(1, MAX_EC_LISTS):
         Tracker.alias(str, 'il{0}pi{1}id'.format(list_index, product_index))  # Product impression SKU
-        Tracker.alias(unicode, 'il{0}pi{1}nm'.format(list_index, product_index))  # Product impression name
-        Tracker.alias(unicode, 'il{0}pi{1}br'.format(list_index, product_index))  # Product impression brand
-        Tracker.alias(unicode, 'il{0}pi{1}ca'.format(list_index, product_index))  # Product impression category
-        Tracker.alias(unicode, 'il{0}pi{1}va'.format(list_index, product_index))  # Product impression variant
+        Tracker.alias(str, 'il{0}pi{1}nm'.format(list_index, product_index))  # Product impression name
+        Tracker.alias(str, 'il{0}pi{1}br'.format(list_index, product_index))  # Product impression brand
+        Tracker.alias(str, 'il{0}pi{1}ca'.format(list_index, product_index))  # Product impression category
+        Tracker.alias(str, 'il{0}pi{1}va'.format(list_index, product_index))  # Product impression variant
         Tracker.alias(int, 'il{0}pi{1}ps'.format(list_index, product_index))  # Product impression position
         Tracker.alias(int, 'il{0}pi{1}pr'.format(list_index, product_index))  # Product impression price
 
@@ -417,11 +408,11 @@ for product_index in range(1, MAX_EC_PRODUCTS):
                                                         custom_index))  # Product impression custom metric
 
 for list_index in range(1, MAX_EC_LISTS):
-    Tracker.alias(unicode, 'il{0}nm'.format(list_index))  # Product impression list name
+    Tracker.alias(str, 'il{0}nm'.format(list_index))  # Product impression list name
 
 for promotion_index in range(1, MAX_EC_PROMOTIONS):
     Tracker.alias(str, 'promo{0}id'.format(promotion_index))  # Promotion ID
-    Tracker.alias(unicode, 'promo{0}nm'.format(promotion_index))  # Promotion name
+    Tracker.alias(str, 'promo{0}nm'.format(promotion_index))  # Promotion name
     Tracker.alias(str, 'promo{0}cr'.format(promotion_index))  # Promotion creative
     Tracker.alias(str, 'promo{0}ps'.format(promotion_index))  # Promotion position
 

lib/UniversalAnalytics/__init__.py

@@ -1 +1 @@
-import Tracker
+from . import Tracker

lib/appdirs.py

@@ -0,0 +1,608 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2005-2010 ActiveState Software Inc.
+# Copyright (c) 2013 Eddy Petrișor
+
+"""Utilities for determining application-specific dirs.
+
+See <http://github.com/ActiveState/appdirs> for details and usage.
+"""
+# Dev Notes:
+# - MSDN on where to store app data files:
+#   http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120
+# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html
+# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
+
+__version_info__ = (1, 4, 3)
+__version__ = '.'.join(map(str, __version_info__))
+
+
+import sys
+import os
+
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    unicode = str
+
+if sys.platform.startswith('java'):
+    import platform
+    os_name = platform.java_ver()[3][0]
+    if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc.
+        system = 'win32'
+    elif os_name.startswith('Mac'): # "Mac OS X", etc.
+        system = 'darwin'
+    else: # "Linux", "SunOS", "FreeBSD", etc.
+        # Setting this to "linux2" is not ideal, but only Windows or Mac
+        # are actually checked for and the rest of the module expects
+        # *sys.platform* style strings.
+        system = 'linux2'
+else:
+    system = sys.platform
+
+
+
+def user_data_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user data directories are:
+        Mac OS X:               ~/Library/Application Support/<AppName>
+        Unix:                   ~/.local/share/<AppName>    # or in $XDG_DATA_HOME, if defined
+        Win XP (not roaming):   C:\Documents and Settings\<username>\Application Data\<AppAuthor>\<AppName>
+        Win XP (roaming):       C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>
+        Win 7  (not roaming):   C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>
+        Win 7  (roaming):       C:\Users\<username>\AppData\Roaming\<AppAuthor>\<AppName>
+
+    For Unix, we follow the XDG spec and support $XDG_DATA_HOME.
+    That means, by default "~/.local/share/<AppName>".
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA"
+        path = os.path.normpath(_get_win_folder(const))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+    elif system == 'darwin':
+        path = os.path.expanduser('~/Library/Application Support/')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def site_data_dir(appname=None, appauthor=None, version=None, multipath=False):
+    r"""Return full path to the user-shared data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "multipath" is an optional parameter only applicable to *nix
+            which indicates that the entire list of data dirs should be
+            returned. By default, the first item from XDG_DATA_DIRS is
+            returned, or '/usr/local/share/<AppName>',
+            if XDG_DATA_DIRS is not set
+
+    Typical site data directories are:
+        Mac OS X:   /Library/Application Support/<AppName>
+        Unix:       /usr/local/share/<AppName> or /usr/share/<AppName>
+        Win XP:     C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName>
+        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
+        Win 7:      C:\ProgramData\<AppAuthor>\<AppName>   # Hidden, but writeable on Win 7.
+
+    For Unix, this is using the $XDG_DATA_DIRS[0] default.
+
+    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA"))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+    elif system == 'darwin':
+        path = os.path.expanduser('/Library/Application Support')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        # XDG default for $XDG_DATA_DIRS
+        # only first, if multipath is False
+        path = os.getenv('XDG_DATA_DIRS',
+                         os.pathsep.join(['/usr/local/share', '/usr/share']))
+        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)]
+        if appname:
+            if version:
+                appname = os.path.join(appname, version)
+            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+
+        if multipath:
+            path = os.pathsep.join(pathlist)
+        else:
+            path = pathlist[0]
+        return path
+
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_config_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific config dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user config directories are:
+        Mac OS X:               same as user_data_dir
+        Unix:                   ~/.config/<AppName>     # or in $XDG_CONFIG_HOME, if defined
+        Win *:                  same as user_data_dir
+
+    For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME.
+    That means, by default "~/.config/<AppName>".
+    """
+    if system in ["win32", "darwin"]:
+        path = user_data_dir(appname, appauthor, None, roaming)
+    else:
+        path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def site_config_dir(appname=None, appauthor=None, version=None, multipath=False):
+    r"""Return full path to the user-shared data dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "multipath" is an optional parameter only applicable to *nix
+            which indicates that the entire list of config dirs should be
+            returned. By default, the first item from XDG_CONFIG_DIRS is
+            returned, or '/etc/xdg/<AppName>', if XDG_CONFIG_DIRS is not set
+
+    Typical site config directories are:
+        Mac OS X:   same as site_data_dir
+        Unix:       /etc/xdg/<AppName> or $XDG_CONFIG_DIRS[i]/<AppName> for each value in
+                    $XDG_CONFIG_DIRS
+        Win *:      same as site_data_dir
+        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
+
+    For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False
+
+    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
+    """
+    if system in ["win32", "darwin"]:
+        path = site_data_dir(appname, appauthor)
+        if appname and version:
+            path = os.path.join(path, version)
+    else:
+        # XDG default for $XDG_CONFIG_DIRS
+        # only first, if multipath is False
+        path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg')
+        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)]
+        if appname:
+            if version:
+                appname = os.path.join(appname, version)
+            pathlist = [os.sep.join([x, appname]) for x in pathlist]
+
+        if multipath:
+            path = os.pathsep.join(pathlist)
+        else:
+            path = pathlist[0]
+    return path
+
+
+def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True):
+    r"""Return full path to the user-specific cache dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "opinion" (boolean) can be False to disable the appending of
+            "Cache" to the base app data dir for Windows. See
+            discussion below.
+
+    Typical user cache directories are:
+        Mac OS X:   ~/Library/Caches/<AppName>
+        Unix:       ~/.cache/<AppName> (XDG default)
+        Win XP:     C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Cache
+        Vista:      C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Cache
+
+    On Windows the only suggestion in the MSDN docs is that local settings go in
+    the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming
+    app data dir (the default returned by `user_data_dir` above). Apps typically
+    put cache data somewhere *under* the given dir here. Some examples:
+        ...\Mozilla\Firefox\Profiles\<ProfileName>\Cache
+        ...\Acme\SuperApp\Cache\1.0
+    OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value.
+    This can be disabled with the `opinion=False` option.
+    """
+    if system == "win32":
+        if appauthor is None:
+            appauthor = appname
+        path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
+        if appname:
+            if appauthor is not False:
+                path = os.path.join(path, appauthor, appname)
+            else:
+                path = os.path.join(path, appname)
+            if opinion:
+                path = os.path.join(path, "Cache")
+    elif system == 'darwin':
+        path = os.path.expanduser('~/Library/Caches')
+        if appname:
+            path = os.path.join(path, appname)
+    else:
+        path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_state_dir(appname=None, appauthor=None, version=None, roaming=False):
+    r"""Return full path to the user-specific state dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "roaming" (boolean, default False) can be set True to use the Windows
+            roaming appdata directory. That means that for users on a Windows
+            network setup for roaming profiles, this user data will be
+            sync'd on login. See
+            <http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx>
+            for a discussion of issues.
+
+    Typical user state directories are:
+        Mac OS X:  same as user_data_dir
+        Unix:      ~/.local/state/<AppName>   # or in $XDG_STATE_HOME, if defined
+        Win *:     same as user_data_dir
+
+    For Unix, we follow this Debian proposal <https://wiki.debian.org/XDGBaseDirectorySpecification#state>
+    to extend the XDG spec and support $XDG_STATE_HOME.
+
+    That means, by default "~/.local/state/<AppName>".
+    """
+    if system in ["win32", "darwin"]:
+        path = user_data_dir(appname, appauthor, None, roaming)
+    else:
+        path = os.getenv('XDG_STATE_HOME', os.path.expanduser("~/.local/state"))
+        if appname:
+            path = os.path.join(path, appname)
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+def user_log_dir(appname=None, appauthor=None, version=None, opinion=True):
+    r"""Return full path to the user-specific log dir for this application.
+
+        "appname" is the name of application.
+            If None, just the system directory is returned.
+        "appauthor" (only used on Windows) is the name of the
+            appauthor or distributing body for this application. Typically
+            it is the owning company name. This falls back to appname. You may
+            pass False to disable it.
+        "version" is an optional version path element to append to the
+            path. You might want to use this if you want multiple versions
+            of your app to be able to run independently. If used, this
+            would typically be "<major>.<minor>".
+            Only applied when appname is present.
+        "opinion" (boolean) can be False to disable the appending of
+            "Logs" to the base app data dir for Windows, and "log" to the
+            base cache dir for Unix. See discussion below.
+
+    Typical user log directories are:
+        Mac OS X:   ~/Library/Logs/<AppName>
+        Unix:       ~/.cache/<AppName>/log  # or under $XDG_CACHE_HOME if defined
+        Win XP:     C:\Documents and Settings\<username>\Local Settings\Application Data\<AppAuthor>\<AppName>\Logs
+        Vista:      C:\Users\<username>\AppData\Local\<AppAuthor>\<AppName>\Logs
+
+    On Windows the only suggestion in the MSDN docs is that local settings
+    go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in
+    examples of what some windows apps use for a logs dir.)
+
+    OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA`
+    value for Windows and appends "log" to the user cache dir for Unix.
+    This can be disabled with the `opinion=False` option.
+    """
+    if system == "darwin":
+        path = os.path.join(
+            os.path.expanduser('~/Library/Logs'),
+            appname)
+    elif system == "win32":
+        path = user_data_dir(appname, appauthor, version)
+        version = False
+        if opinion:
+            path = os.path.join(path, "Logs")
+    else:
+        path = user_cache_dir(appname, appauthor, version)
+        version = False
+        if opinion:
+            path = os.path.join(path, "log")
+    if appname and version:
+        path = os.path.join(path, version)
+    return path
+
+
+class AppDirs(object):
+    """Convenience wrapper for getting application dirs."""
+    def __init__(self, appname=None, appauthor=None, version=None,
+            roaming=False, multipath=False):
+        self.appname = appname
+        self.appauthor = appauthor
+        self.version = version
+        self.roaming = roaming
+        self.multipath = multipath
+
+    @property
+    def user_data_dir(self):
+        return user_data_dir(self.appname, self.appauthor,
+                             version=self.version, roaming=self.roaming)
+
+    @property
+    def site_data_dir(self):
+        return site_data_dir(self.appname, self.appauthor,
+                             version=self.version, multipath=self.multipath)
+
+    @property
+    def user_config_dir(self):
+        return user_config_dir(self.appname, self.appauthor,
+                               version=self.version, roaming=self.roaming)
+
+    @property
+    def site_config_dir(self):
+        return site_config_dir(self.appname, self.appauthor,
+                             version=self.version, multipath=self.multipath)
+
+    @property
+    def user_cache_dir(self):
+        return user_cache_dir(self.appname, self.appauthor,
+                              version=self.version)
+
+    @property
+    def user_state_dir(self):
+        return user_state_dir(self.appname, self.appauthor,
+                              version=self.version)
+
+    @property
+    def user_log_dir(self):
+        return user_log_dir(self.appname, self.appauthor,
+                            version=self.version)
+
+
+#---- internal support stuff
+
+def _get_win_folder_from_registry(csidl_name):
+    """This is a fallback technique at best. I'm not sure if using the
+    registry for this guarantees us the correct answer for all CSIDL_*
+    names.
+    """
+    if PY3:
+      import winreg as _winreg
+    else:
+      import _winreg
+
+    shell_folder_name = {
+        "CSIDL_APPDATA": "AppData",
+        "CSIDL_COMMON_APPDATA": "Common AppData",
+        "CSIDL_LOCAL_APPDATA": "Local AppData",
+    }[csidl_name]
+
+    key = _winreg.OpenKey(
+        _winreg.HKEY_CURRENT_USER,
+        r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+    )
+    dir, type = _winreg.QueryValueEx(key, shell_folder_name)
+    return dir
+
+
+def _get_win_folder_with_pywin32(csidl_name):
+    from win32com.shell import shellcon, shell
+    dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0)
+    # Try to make this a unicode path because SHGetFolderPath does
+    # not return unicode strings when there is unicode data in the
+    # path.
+    try:
+        dir = unicode(dir)
+
+        # Downgrade to short path name if have highbit chars. See
+        # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+        has_high_char = False
+        for c in dir:
+            if ord(c) > 255:
+                has_high_char = True
+                break
+        if has_high_char:
+            try:
+                import win32api
+                dir = win32api.GetShortPathName(dir)
+            except ImportError:
+                pass
+    except UnicodeError:
+        pass
+    return dir
+
+
+def _get_win_folder_with_ctypes(csidl_name):
+    import ctypes
+
+    csidl_const = {
+        "CSIDL_APPDATA": 26,
+        "CSIDL_COMMON_APPDATA": 35,
+        "CSIDL_LOCAL_APPDATA": 28,
+    }[csidl_name]
+
+    buf = ctypes.create_unicode_buffer(1024)
+    ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
+
+    # Downgrade to short path name if have highbit chars. See
+    # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+    has_high_char = False
+    for c in buf:
+        if ord(c) > 255:
+            has_high_char = True
+            break
+    if has_high_char:
+        buf2 = ctypes.create_unicode_buffer(1024)
+        if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024):
+            buf = buf2
+
+    return buf.value
+
+def _get_win_folder_with_jna(csidl_name):
+    import array
+    from com.sun import jna
+    from com.sun.jna.platform import win32
+
+    buf_size = win32.WinDef.MAX_PATH * 2
+    buf = array.zeros('c', buf_size)
+    shell = win32.Shell32.INSTANCE
+    shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf)
+    dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+    # Downgrade to short path name if have highbit chars. See
+    # <http://bugs.activestate.com/show_bug.cgi?id=85099>.
+    has_high_char = False
+    for c in dir:
+        if ord(c) > 255:
+            has_high_char = True
+            break
+    if has_high_char:
+        buf = array.zeros('c', buf_size)
+        kernel = win32.Kernel32.INSTANCE
+        if kernel.GetShortPathName(dir, buf, buf_size):
+            dir = jna.Native.toString(buf.tostring()).rstrip("\0")
+
+    return dir
+
+if system == "win32":
+    try:
+        import win32com.shell
+        _get_win_folder = _get_win_folder_with_pywin32
+    except ImportError:
+        try:
+            from ctypes import windll
+            _get_win_folder = _get_win_folder_with_ctypes
+        except ImportError:
+            try:
+                import com.sun.jna
+                _get_win_folder = _get_win_folder_with_jna
+            except ImportError:
+                _get_win_folder = _get_win_folder_from_registry
+
+
+#---- self test code
+
+if __name__ == "__main__":
+    appname = "MyApp"
+    appauthor = "MyCompany"
+
+    props = ("user_data_dir",
+             "user_config_dir",
+             "user_cache_dir",
+             "user_state_dir",
+             "user_log_dir",
+             "site_data_dir",
+             "site_config_dir")
+
+    print("-- app dirs %s --" % __version__)
+
+    print("-- app dirs (with optional 'version')")
+    dirs = AppDirs(appname, appauthor, version="1.0")
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (without optional 'version')")
+    dirs = AppDirs(appname, appauthor)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (without optional 'appauthor')")
+    dirs = AppDirs(appname)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))
+
+    print("\n-- app dirs (with disabled 'appauthor')")
+    dirs = AppDirs(appname, appauthor=False)
+    for prop in props:
+        print("%s: %s" % (prop, getattr(dirs, prop)))

lib/apscheduler/events.py

@@ -3,7 +3,7 @@ __all__ = ('EVENT_SCHEDULER_STARTED', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_SCHEDUL
            'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED',
            'EVENT_JOB_ADDED', 'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED',
            'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED', 'EVENT_JOB_SUBMITTED', 'EVENT_JOB_MAX_INSTANCES',
-           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent')
+           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent', 'JobSubmissionEvent')
 
 
 EVENT_SCHEDULER_STARTED = EVENT_SCHEDULER_START = 2 ** 0

lib/apscheduler/executors/asyncio.py

@@ -3,12 +3,11 @@ from __future__ import absolute_import
 import sys
 
 from apscheduler.executors.base import BaseExecutor, run_job
+from apscheduler.util import iscoroutinefunction_partial
 
 try:
-    from asyncio import iscoroutinefunction
     from apscheduler.executors.base_py3 import run_coroutine_job
 except ImportError:
-    from trollius import iscoroutinefunction
     run_coroutine_job = None
 
 
@@ -46,7 +45,7 @@ class AsyncIOExecutor(BaseExecutor):
             else:
                 self._run_job_success(job.id, events)
 
-        if iscoroutinefunction(job.func):
+        if iscoroutinefunction_partial(job.func):
             if run_coroutine_job is not None:
                 coro = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
                 f = self._eventloop.create_task(coro)

lib/apscheduler/executors/tornado.py

@@ -8,10 +8,10 @@ from tornado.gen import convert_yielded
 from apscheduler.executors.base import BaseExecutor, run_job
 
 try:
-    from inspect import iscoroutinefunction
     from apscheduler.executors.base_py3 import run_coroutine_job
+    from apscheduler.util import iscoroutinefunction_partial
 except ImportError:
-    def iscoroutinefunction(func):
+    def iscoroutinefunction_partial(func):
         return False
 
 
@@ -44,7 +44,7 @@ class TornadoExecutor(BaseExecutor):
             else:
                 self._run_job_success(job.id, events)
 
-        if iscoroutinefunction(job.func):
+        if iscoroutinefunction_partial(job.func):
             f = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
         else:
             f = self.executor.submit(run_job, job, job._jobstore_alias, run_times,

lib/apscheduler/job.py

@@ -1,4 +1,4 @@
-from collections import Iterable, Mapping
+from inspect import ismethod, isclass
 from uuid import uuid4
 
 import six
@@ -8,6 +8,11 @@ from apscheduler.util import (
     ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args,
     convert_to_datetime)
 
+try:
+    from collections.abc import Iterable, Mapping
+except ImportError:
+    from collections import Iterable, Mapping
+
 
 class Job(object):
     """
@@ -235,13 +240,20 @@ class Job(object):
                 'be determined. Consider giving a textual reference (module:function name) '
                 'instead.' % (self.func,))
 
+        # Instance methods cannot survive serialization as-is, so store the "self" argument
+        # explicitly
+        if ismethod(self.func) and not isclass(self.func.__self__):
+            args = (self.func.__self__,) + tuple(self.args)
+        else:
+            args = self.args
+
         return {
             'version': 1,
             'id': self.id,
             'func': self.func_ref,
             'trigger': self.trigger,
             'executor': self.executor,
-            'args': self.args,
+            'args': args,
             'kwargs': self.kwargs,
             'name': self.name,
             'misfire_grace_time': self.misfire_grace_time,

lib/apscheduler/jobstores/redis.py

@@ -14,7 +14,7 @@ except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from redis import StrictRedis
+    from redis import Redis
 except ImportError:  # pragma: nocover
     raise ImportError('RedisJobStore requires redis installed')
 
@@ -47,7 +47,7 @@ class RedisJobStore(BaseJobStore):
         self.pickle_protocol = pickle_protocol
         self.jobs_key = jobs_key
         self.run_times_key = run_times_key
-        self.redis = StrictRedis(db=int(db), **connect_args)
+        self.redis = Redis(db=int(db), **connect_args)
 
     def lookup_job(self, job_id):
         job_state = self.redis.hget(self.jobs_key, job_id)
@@ -81,7 +81,9 @@ class RedisJobStore(BaseJobStore):
             pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
                                                           self.pickle_protocol))
             if job.next_run_time:
-                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
+
             pipe.execute()
 
     def update_job(self, job):
@@ -92,9 +94,11 @@ class RedisJobStore(BaseJobStore):
             pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
                                                           self.pickle_protocol))
             if job.next_run_time:
-                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
             else:
                 pipe.zrem(self.run_times_key, job.id)
+
             pipe.execute()
 
     def remove_job(self, job_id):

lib/apscheduler/jobstores/rethinkdb.py

@@ -10,7 +10,7 @@ except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    import rethinkdb as r
+    from rethinkdb import RethinkDB
 except ImportError:  # pragma: nocover
     raise ImportError('RethinkDBJobStore requires rethinkdb installed')
 
@@ -40,10 +40,12 @@ class RethinkDBJobStore(BaseJobStore):
             raise ValueError('The "table" parameter must not be empty')
 
         self.database = database
-        self.table = table
+        self.table_name = table
+        self.table = None
         self.client = client
         self.pickle_protocol = pickle_protocol
         self.connect_args = connect_args
+        self.r = RethinkDB()
         self.conn = None
 
     def start(self, scheduler, alias):
@@ -52,31 +54,31 @@ class RethinkDBJobStore(BaseJobStore):
         if self.client:
             self.conn = maybe_ref(self.client)
         else:
-            self.conn = r.connect(db=self.database, **self.connect_args)
+            self.conn = self.r.connect(db=self.database, **self.connect_args)
 
-        if self.database not in r.db_list().run(self.conn):
-            r.db_create(self.database).run(self.conn)
+        if self.database not in self.r.db_list().run(self.conn):
+            self.r.db_create(self.database).run(self.conn)
 
-        if self.table not in r.table_list().run(self.conn):
-            r.table_create(self.table).run(self.conn)
+        if self.table_name not in self.r.table_list().run(self.conn):
+            self.r.table_create(self.table_name).run(self.conn)
 
-        if 'next_run_time' not in r.table(self.table).index_list().run(self.conn):
-            r.table(self.table).index_create('next_run_time').run(self.conn)
+        if 'next_run_time' not in self.r.table(self.table_name).index_list().run(self.conn):
+            self.r.table(self.table_name).index_create('next_run_time').run(self.conn)
 
-        self.table = r.db(self.database).table(self.table)
+        self.table = self.r.db(self.database).table(self.table_name)
 
     def lookup_job(self, job_id):
         results = list(self.table.get_all(job_id).pluck('job_state').run(self.conn))
         return self._reconstitute_job(results[0]['job_state']) if results else None
 
     def get_due_jobs(self, now):
-        return self._get_jobs(r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
+        return self._get_jobs(self.r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
 
     def get_next_run_time(self):
         results = list(
             self.table
-            .filter(r.row['next_run_time'] != None)  # flake8: noqa
-            .order_by(r.asc('next_run_time'))
+            .filter(self.r.row['next_run_time'] != None)  # noqa
+            .order_by(self.r.asc('next_run_time'))
             .map(lambda x: x['next_run_time'])
             .limit(1)
             .run(self.conn)
@@ -92,7 +94,7 @@ class RethinkDBJobStore(BaseJobStore):
         job_dict = {
             'id': job.id,
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
-            'job_state': r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
         }
         results = self.table.insert(job_dict).run(self.conn)
         if results['errors'] > 0:
@@ -101,7 +103,7 @@ class RethinkDBJobStore(BaseJobStore):
     def update_job(self, job):
         changes = {
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
-            'job_state': r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
         }
         results = self.table.get_all(job.id).update(changes).run(self.conn)
         skipped = False in map(lambda x: results[x] == 0, results.keys())
@@ -130,20 +132,20 @@ class RethinkDBJobStore(BaseJobStore):
     def _get_jobs(self, predicate=None):
         jobs = []
         failed_job_ids = []
-        query = (self.table.filter(r.row['next_run_time'] != None).filter(predicate) if
-                 predicate else self.table)
+        query = (self.table.filter(self.r.row['next_run_time'] != None).filter(predicate)  # noqa
+                 if predicate else self.table)
         query = query.order_by('next_run_time', 'id').pluck('id', 'job_state')
 
         for document in query.run(self.conn):
             try:
                 jobs.append(self._reconstitute_job(document['job_state']))
-            except:
+            except Exception:
                 self._logger.exception('Unable to restore job "%s" -- removing it', document['id'])
                 failed_job_ids.append(document['id'])
 
         # Remove all the jobs we failed to restore
         if failed_job_ids:
-            r.expr(failed_job_ids).for_each(
+            self.r.expr(failed_job_ids).for_each(
                 lambda job_id: self.table.get_all(job_id).delete()).run(self.conn)
 
         return jobs

lib/apscheduler/jobstores/sqlalchemy.py

@@ -106,7 +106,7 @@ class SQLAlchemyJobStore(BaseJobStore):
         }).where(self.jobs_t.c.id == job.id)
         result = self.engine.execute(update)
         if result.rowcount == 0:
-            raise JobLookupError(id)
+            raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
         delete = self.jobs_t.delete().where(self.jobs_t.c.id == job_id)

lib/apscheduler/schedulers/base.py

@@ -1,7 +1,6 @@
 from __future__ import print_function
 
 from abc import ABCMeta, abstractmethod
-from collections import MutableMapping
 from threading import RLock
 from datetime import datetime, timedelta
 from logging import getLogger
@@ -19,13 +18,19 @@ from apscheduler.jobstores.base import ConflictingIdError, JobLookupError, BaseJ
 from apscheduler.jobstores.memory import MemoryJobStore
 from apscheduler.job import Job
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import asbool, asint, astimezone, maybe_ref, timedelta_seconds, undefined
+from apscheduler.util import (
+    asbool, asint, astimezone, maybe_ref, timedelta_seconds, undefined, TIMEOUT_MAX)
 from apscheduler.events import (
     SchedulerEvent, JobEvent, JobSubmissionEvent, EVENT_SCHEDULER_START, EVENT_SCHEDULER_SHUTDOWN,
     EVENT_JOBSTORE_ADDED, EVENT_JOBSTORE_REMOVED, EVENT_ALL, EVENT_JOB_MODIFIED, EVENT_JOB_REMOVED,
     EVENT_JOB_ADDED, EVENT_EXECUTOR_ADDED, EVENT_EXECUTOR_REMOVED, EVENT_ALL_JOBS_REMOVED,
     EVENT_JOB_SUBMITTED, EVENT_JOB_MAX_INSTANCES, EVENT_SCHEDULER_RESUMED, EVENT_SCHEDULER_PAUSED)
 
+try:
+    from collections.abc import MutableMapping
+except ImportError:
+    from collections import MutableMapping
+
 #: constant indicating a scheduler's stopped state
 STATE_STOPPED = 0
 #: constant indicating a scheduler's running state (started and processing jobs)
@@ -126,11 +131,14 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         :param bool paused: if ``True``, don't start job processing until :meth:`resume` is called
         :raises SchedulerAlreadyRunningError: if the scheduler is already running
+        :raises RuntimeError: if running under uWSGI with threads disabled
 
         """
         if self.state != STATE_STOPPED:
             raise SchedulerAlreadyRunningError
 
+        self._check_uwsgi()
+
         with self._executors_lock:
             # Create a default executor if nothing else is configured
             if 'default' not in self._executors:
@@ -177,12 +185,13 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         self.state = STATE_STOPPED
 
-        with self._jobstores_lock, self._executors_lock:
-            # Shut down all executors
+        # Shut down all executors
+        with self._executors_lock:
             for executor in six.itervalues(self._executors):
                 executor.shutdown(wait)
 
-            # Shut down all job stores
+        # Shut down all job stores
+        with self._jobstores_lock:
             for jobstore in six.itervalues(self._jobstores):
                 jobstore.shutdown()
 
@@ -546,7 +555,7 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         """
         if pending is not None:
             warnings.warn('The "pending" option is deprecated -- get_jobs() always returns '
-                          'pending jobs if the scheduler has been started and scheduled jobs '
+                          'scheduled jobs if the scheduler has been started and pending jobs '
                           'otherwise', DeprecationWarning)
 
         with self._jobstores_lock:
@@ -589,14 +598,13 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         """
         jobstore_alias = None
         with self._jobstores_lock:
+            # Check if the job is among the pending jobs
             if self.state == STATE_STOPPED:
-                # Check if the job is among the pending jobs
-                if self.state == STATE_STOPPED:
-                    for i, (job, alias, replace_existing) in enumerate(self._pending_jobs):
-                        if job.id == job_id and jobstore in (None, alias):
-                            del self._pending_jobs[i]
-                            jobstore_alias = alias
-                            break
+                for i, (job, alias, replace_existing) in enumerate(self._pending_jobs):
+                    if job.id == job_id and jobstore in (None, alias):
+                        del self._pending_jobs[i]
+                        jobstore_alias = alias
+                        break
             else:
                 # Otherwise, try to remove it from each store until it succeeds or we run out of
                 # stores to check
@@ -824,6 +832,14 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
                 except BaseException:
                     self._logger.exception('Error notifying listener')
 
+    def _check_uwsgi(self):
+        """Check if we're running under uWSGI with threads disabled."""
+        uwsgi_module = sys.modules.get('uwsgi')
+        if not getattr(uwsgi_module, 'has_threads', True):
+            raise RuntimeError('The scheduler seems to be running under uWSGI, but threads have '
+                               'been disabled. You must run uWSGI with the --enable-threads '
+                               'option for the scheduler to work.')
+
     def _real_add_job(self, job, jobstore_alias, replace_existing):
         """
         :param Job job: the job to add
@@ -999,7 +1015,7 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
             wait_seconds = None
             self._logger.debug('No jobs; waiting until a job is added')
         else:
-            wait_seconds = max(timedelta_seconds(next_wakeup_time - now), 0)
+            wait_seconds = min(max(timedelta_seconds(next_wakeup_time - now), 0), TIMEOUT_MAX)
             self._logger.debug('Next wakeup is due at %s (in %f seconds)', next_wakeup_time,
                                wait_seconds)
 

lib/apscheduler/schedulers/qt.py

@@ -9,7 +9,7 @@ except (ImportError, RuntimeError):  # pragma: nocover
         from PyQt4.QtCore import QObject, QTimer
     except ImportError:
         try:
-            from PySide.QtCore import QObject, QTimer  # flake8: noqa
+            from PySide.QtCore import QObject, QTimer  # noqa
         except ImportError:
             raise ImportError('QtScheduler requires either PyQt5, PyQt4 or PySide installed')
 
@@ -26,7 +26,8 @@ class QtScheduler(BaseScheduler):
     def _start_timer(self, wait_seconds):
         self._stop_timer()
         if wait_seconds is not None:
-            self._timer = QTimer.singleShot(wait_seconds * 1000, self._process_jobs)
+            wait_time = min(wait_seconds * 1000, 2147483647)
+            self._timer = QTimer.singleShot(wait_time, self._process_jobs)
 
     def _stop_timer(self):
         if self._timer:

lib/apscheduler/triggers/cron/__init__.py

@@ -192,9 +192,8 @@ class CronTrigger(BaseTrigger):
                 return None
 
         if fieldnum >= 0:
-            if self.jitter is not None:
-                next_date = self._apply_jitter(next_date, self.jitter, now)
-            return next_date
+            next_date = self._apply_jitter(next_date, self.jitter, now)
+            return min(next_date, self.end_date) if self.end_date else next_date
 
     def __getstate__(self):
         return {

lib/apscheduler/triggers/cron/expressions.py

@@ -9,7 +9,7 @@ __all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression',
            'WeekdayPositionExpression', 'LastDayOfMonthExpression')
 
 
-WEEKDAYS = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']
+WEEKDAYS = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
 MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
 
 

lib/apscheduler/triggers/cron/fields.py

@@ -104,7 +104,7 @@ class DayOfWeekField(BaseField):
     COMPILERS = BaseField.COMPILERS + [WeekdayRangeExpression]
 
     def get_value(self, dateval):
-        return dateval.isoweekday() % 7
+        return dateval.weekday()
 
 
 class MonthField(BaseField):

lib/apscheduler/util.py

@@ -1,12 +1,14 @@
 """This module contains several handy functions primarily meant for internal use."""
 
 from __future__ import division
+
 from datetime import date, datetime, time, timedelta, tzinfo
 from calendar import timegm
-import re
 from functools import partial
+from inspect import isclass, ismethod
+import re
 
-from pytz import timezone, utc
+from pytz import timezone, utc, FixedOffset
 import six
 
 try:
@@ -19,9 +21,19 @@ try:
 except ImportError:
     TIMEOUT_MAX = 4294967  # Maximum value accepted by Event.wait() on Windows
 
+try:
+    from asyncio import iscoroutinefunction
+except ImportError:
+    try:
+        from trollius import iscoroutinefunction
+    except ImportError:
+        def iscoroutinefunction(func):
+            return False
+
 __all__ = ('asint', 'asbool', 'astimezone', 'convert_to_datetime', 'datetime_to_utc_timestamp',
            'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name',
-           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args')
+           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args',
+           'TIMEOUT_MAX')
 
 
 class _Undefined(object):
@@ -92,8 +104,9 @@ def astimezone(obj):
 
 _DATE_REGEX = re.compile(
     r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})'
-    r'(?: (?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
-    r'(?:\.(?P<microsecond>\d{1,6}))?)?')
+    r'(?:[ T](?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
+    r'(?:\.(?P<microsecond>\d{1,6}))?'
+    r'(?P<timezone>Z|[+-]\d\d:\d\d)?)?$')
 
 
 def convert_to_datetime(input, tz, arg_name):
@@ -105,7 +118,9 @@ def convert_to_datetime(input, tz, arg_name):
     If the input is a string, it is parsed as a datetime with the given timezone.
 
     Date strings are accepted in three different forms: date only (Y-m-d), date with time
-    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro).
+    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro). Additionally you can
+    override the time zone by giving a specific offset in the format specified by ISO 8601:
+    Z (UTC), +HH:MM or -HH:MM.
 
     :param str|datetime input: the datetime or string to convert to a timezone aware datetime
     :param datetime.tzinfo tz: timezone to interpret ``input`` in
@@ -123,8 +138,17 @@ def convert_to_datetime(input, tz, arg_name):
         m = _DATE_REGEX.match(input)
         if not m:
             raise ValueError('Invalid date string')
-        values = [(k, int(v or 0)) for k, v in m.groupdict().items()]
-        values = dict(values)
+
+        values = m.groupdict()
+        tzname = values.pop('timezone')
+        if tzname == 'Z':
+            tz = utc
+        elif tzname:
+            hours, minutes = (int(x) for x in tzname[1:].split(':'))
+            sign = 1 if tzname[0] == '+' else -1
+            tz = FixedOffset(sign * (hours * 60 + minutes))
+
+        values = {k: int(v or 0) for k, v in values.items()}
         datetime_ = datetime(**values)
     else:
         raise TypeError('Unsupported type for %s: %s' % (arg_name, input.__class__.__name__))
@@ -210,7 +234,7 @@ def get_callable_name(func):
     # class methods, bound and unbound methods
     f_self = getattr(func, '__self__', None) or getattr(func, 'im_self', None)
     if f_self and hasattr(func, '__name__'):
-        f_class = f_self if isinstance(f_self, type) else f_self.__class__
+        f_class = f_self if isclass(f_self) else f_self.__class__
     else:
         f_class = getattr(func, 'im_class', None)
 
@@ -248,7 +272,18 @@ def obj_to_ref(obj):
     if '<locals>' in name:
         raise ValueError('Cannot create a reference to a nested function')
 
-    return '%s:%s' % (obj.__module__, name)
+    if ismethod(obj):
+        if hasattr(obj, 'im_self') and obj.im_self:
+            # bound method
+            module = obj.im_self.__module__
+        elif hasattr(obj, 'im_class') and obj.im_class:
+            # unbound method
+            module = obj.im_class.__module__
+        else:
+            module = obj.__module__
+    else:
+        module = obj.__module__
+    return '%s:%s' % (module, name)
 
 
 def ref_to_obj(ref):
@@ -383,3 +418,12 @@ def check_callable_args(func, args, kwargs):
         raise ValueError(
             'The target callable does not accept the following keyword arguments: %s' %
             ', '.join(unmatched_kwargs))
+
+
+def iscoroutinefunction_partial(f):
+    while isinstance(f, partial):
+        f = f.func
+
+    # The asyncio version of iscoroutinefunction includes testing for @coroutine
+    # decorations vs. the inspect version which does not.
+    return iscoroutinefunction(f)

lib/argparse.py

@@ -1,4 +1,5 @@
 # Author: Steven J. Bethard <steven.bethard@gmail.com>.
+# Maintainer: Thomas Waldmann <tw@waldmann-edv.de>
 
 """Command-line parsing library
 
@@ -61,7 +62,12 @@ considered public as object names -- the API of the formatter objects is
 still considered an implementation detail.)
 """
 
-__version__ = '1.1'
+__version__ = '1.4.0'  # we use our own version number independant of the
+                       # one in stdlib and we release this on pypi.
+
+__external_lib__ = True  # to make sure the tests really test THIS lib,
+                         # not the builtin one in Python stdlib
+
 __all__ = [
     'ArgumentParser',
     'ArgumentError',
@@ -71,7 +77,6 @@ __all__ = [
     'ArgumentDefaultsHelpFormatter',
     'RawDescriptionHelpFormatter',
     'RawTextHelpFormatter',
-    'MetavarTypeHelpFormatter',
     'Namespace',
     'Action',
     'ONE_OR_MORE',
@@ -83,14 +88,35 @@ __all__ = [
 ]
 
 
-import collections as _collections
 import copy as _copy
 import os as _os
 import re as _re
 import sys as _sys
 import textwrap as _textwrap
 
-from gettext import gettext as _, ngettext
+from gettext import gettext as _
+
+try:
+    set
+except NameError:
+    # for python < 2.4 compatibility (sets module is there since 2.3):
+    from sets import Set as set
+
+try:
+    basestring
+except NameError:
+    basestring = str
+
+try:
+    sorted
+except NameError:
+    # for python < 2.4 compatibility:
+    def sorted(iterable, reverse=False):
+        result = list(iterable)
+        result.sort()
+        if reverse:
+            result.reverse()
+        return result
 
 
 def _callable(obj):
@@ -424,8 +450,7 @@ class HelpFormatter(object):
 
             # produce all arg strings
             elif not action.option_strings:
-                default = self._get_default_metavar_for_positional(action)
-                part = self._format_args(action, default)
+                part = self._format_args(action, action.dest)
 
                 # if it's in a group, strip the outer []
                 if action in group_actions:
@@ -447,7 +472,7 @@ class HelpFormatter(object):
                 # if the Optional takes a value, format is:
                 #    -s ARGS or --long ARGS
                 else:
-                    default = self._get_default_metavar_for_optional(action)
+                    default = action.dest.upper()
                     args_string = self._format_args(action, default)
                     part = '%s %s' % (option_string, args_string)
 
@@ -533,8 +558,7 @@ class HelpFormatter(object):
 
     def _format_action_invocation(self, action):
         if not action.option_strings:
-            default = self._get_default_metavar_for_positional(action)
-            metavar, = self._metavar_formatter(action, default)(1)
+            metavar, = self._metavar_formatter(action, action.dest)(1)
             return metavar
 
         else:
@@ -548,7 +572,7 @@ class HelpFormatter(object):
             # if the Optional takes a value, format is:
             #    -s ARGS, --long ARGS
             else:
-                default = self._get_default_metavar_for_optional(action)
+                default = action.dest.upper()
                 args_string = self._format_args(action, default)
                 for option_string in action.option_strings:
                     parts.append('%s %s' % (option_string, args_string))
@@ -626,12 +650,6 @@ class HelpFormatter(object):
     def _get_help_string(self, action):
         return action.help
 
-    def _get_default_metavar_for_optional(self, action):
-        return action.dest.upper()
-
-    def _get_default_metavar_for_positional(self, action):
-        return action.dest
-
 
 class RawDescriptionHelpFormatter(HelpFormatter):
     """Help message formatter which retains any formatting in descriptions.
@@ -672,22 +690,6 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
         return help
 
 
-class MetavarTypeHelpFormatter(HelpFormatter):
-    """Help message formatter which uses the argument 'type' as the default
-    metavar value (instead of the argument 'dest')
-
-    Only the name of this class is considered a public API. All the methods
-    provided by the class are considered an implementation detail.
-    """
-
-    def _get_default_metavar_for_optional(self, action):
-        return action.type.__name__
-
-    def _get_default_metavar_for_positional(self, action):
-        return action.type.__name__
-
-
-
 # =====================
 # Options and Arguments
 # =====================
@@ -1055,7 +1057,7 @@ class _SubParsersAction(Action):
                 metavar += ' (%s)' % ', '.join(aliases)
             sup = super(_SubParsersAction._ChoicesPseudoAction, self)
             sup.__init__(option_strings=[], dest=dest, help=help,
-                         metavar=metavar)
+                        metavar=metavar)
 
     def __init__(self,
                  option_strings,
@@ -1067,7 +1069,7 @@ class _SubParsersAction(Action):
 
         self._prog_prefix = prog
         self._parser_class = parser_class
-        self._name_parser_map = _collections.OrderedDict()
+        self._name_parser_map = {}
         self._choices_actions = []
 
         super(_SubParsersAction, self).__init__(
@@ -1116,9 +1118,8 @@ class _SubParsersAction(Action):
         try:
             parser = self._name_parser_map[parser_name]
         except KeyError:
-            args = {'parser_name': parser_name,
-                    'choices': ', '.join(self._name_parser_map)}
-            msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args
+            tup = parser_name, ', '.join(self._name_parser_map)
+            msg = _('unknown parser %r (choices: %s)' % tup)
             raise ArgumentError(self, msg)
 
         # parse all the remaining options into the namespace
@@ -1147,7 +1148,7 @@ class FileType(object):
             the builtin open() function.
     """
 
-    def __init__(self, mode='r', bufsize=-1):
+    def __init__(self, mode='r', bufsize=None):
         self._mode = mode
         self._bufsize = bufsize
 
@@ -1159,19 +1160,23 @@ class FileType(object):
             elif 'w' in self._mode:
                 return _sys.stdout
             else:
-                msg = _('argument "-" with mode %r') % self._mode
+                msg = _('argument "-" with mode %r' % self._mode)
                 raise ValueError(msg)
 
-        # all other arguments are used as file names
         try:
-            return open(string, self._mode, self._bufsize)
-        except IOError as e:
+            # all other arguments are used as file names
+            if self._bufsize:
+                return open(string, self._mode, self._bufsize)
+            else:
+                return open(string, self._mode)
+        except IOError:
+            err = _sys.exc_info()[1]
             message = _("can't open '%s': %s")
-            raise ArgumentTypeError(message % (string, e))
+            raise ArgumentTypeError(message % (string, err))
 
     def __repr__(self):
-        args = self._mode, self._bufsize
-        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
+        args = [self._mode, self._bufsize]
+        args_str = ', '.join([repr(arg) for arg in args if arg is not None])
         return '%s(%s)' % (type(self).__name__, args_str)
 
 # ===========================
@@ -1189,6 +1194,8 @@ class Namespace(_AttributeHolder):
         for name in kwargs:
             setattr(self, name, kwargs[name])
 
+    __hash__ = None
+
     def __eq__(self, other):
         return vars(self) == vars(other)
 
@@ -1312,20 +1319,13 @@ class _ActionsContainer(object):
         # create the action object, and add it to the parser
         action_class = self._pop_action_class(kwargs)
         if not _callable(action_class):
-            raise ValueError('unknown action "%s"' % (action_class,))
+            raise ValueError('unknown action "%s"' % action_class)
         action = action_class(**kwargs)
 
         # raise an error if the action type is not callable
         type_func = self._registry_get('type', action.type, action.type)
         if not _callable(type_func):
-            raise ValueError('%r is not callable' % (type_func,))
-
-        # raise an error if the metavar does not match the type
-        if hasattr(self, "_get_formatter"):
-            try:
-                self._get_formatter()._format_args(action, None)
-            except TypeError:
-                raise ValueError("length of metavar tuple does not match nargs")
+            raise ValueError('%r is not callable' % type_func)
 
         return self._add_action(action)
 
@@ -1426,11 +1426,10 @@ class _ActionsContainer(object):
         for option_string in args:
             # error on strings that don't start with an appropriate prefix
             if not option_string[0] in self.prefix_chars:
-                args = {'option': option_string,
-                        'prefix_chars': self.prefix_chars}
-                msg = _('invalid option string %(option)r: '
-                        'must start with a character %(prefix_chars)r')
-                raise ValueError(msg % args)
+                msg = _('invalid option string %r: '
+                        'must start with a character %r')
+                tup = option_string, self.prefix_chars
+                raise ValueError(msg % tup)
 
             # strings starting with two prefix characters are long options
             option_strings.append(option_string)
@@ -1483,9 +1482,7 @@ class _ActionsContainer(object):
             conflict_handler(action, confl_optionals)
 
     def _handle_conflict_error(self, action, conflicting_actions):
-        message = ngettext('conflicting option string: %s',
-                           'conflicting option strings: %s',
-                           len(conflicting_actions))
+        message = _('conflicting option string(s): %s')
         conflict_string = ', '.join([option_string
                                      for option_string, action
                                      in conflicting_actions])
@@ -1528,7 +1525,6 @@ class _ArgumentGroup(_ActionsContainer):
         self._defaults = container._defaults
         self._has_negative_number_optionals = \
             container._has_negative_number_optionals
-        self._mutually_exclusive_groups = container._mutually_exclusive_groups
 
     def _add_action(self, action):
         action = super(_ArgumentGroup, self)._add_action(action)
@@ -1630,7 +1626,10 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
 
         # add help and version arguments if necessary
         # (using explicit default to override global argument_default)
-        default_prefix = '-' if '-' in prefix_chars else prefix_chars[0]
+        if '-' in prefix_chars:
+            default_prefix = '-'
+        else:
+            default_prefix = prefix_chars[0]
         if self.add_help:
             self.add_argument(
                 default_prefix+'h', default_prefix*2+'help',
@@ -1743,10 +1742,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
             if action.dest is not SUPPRESS:
                 if not hasattr(namespace, action.dest):
                     if action.default is not SUPPRESS:
-                        default = action.default
-                        if isinstance(action.default, str):
-                            default = self._get_value(action, default)
-                        setattr(namespace, action.dest, default)
+                        setattr(namespace, action.dest, action.default)
 
         # add any parser defaults that aren't present
         for dest in self._defaults:
@@ -1969,12 +1965,28 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         # if we didn't consume all the argument strings, there were extras
         extras.extend(arg_strings[stop_index:])
 
-        # make sure all required actions were present
-        required_actions = [_get_action_name(action) for action in self._actions
-                            if action.required and action not in seen_actions]
-        if required_actions:
-            self.error(_('the following arguments are required: %s') %
-                       ', '.join(required_actions))
+        # if we didn't use all the Positional objects, there were too few
+        # arg strings supplied.
+        if positionals:
+            self.error(_('too few arguments'))
+
+        # make sure all required actions were present, and convert defaults.
+        for action in self._actions:
+            if action not in seen_actions:
+                if action.required:
+                    name = _get_action_name(action)
+                    self.error(_('argument %s is required') % name)
+                else:
+                    # Convert action default now instead of doing it before
+                    # parsing arguments to avoid calling convert functions
+                    # twice (which may fail) if the argument was given, but
+                    # only if it was defined already in the namespace
+                    if (action.default is not None and
+                            isinstance(action.default, basestring) and
+                            hasattr(namespace, action.dest) and
+                            action.default is getattr(namespace, action.dest)):
+                        setattr(namespace, action.dest,
+                                self._get_value(action, action.default))
 
         # make sure all required groups had one option present
         for group in self._mutually_exclusive_groups:
@@ -2038,9 +2050,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 OPTIONAL: _('expected at most one argument'),
                 ONE_OR_MORE: _('expected at least one argument'),
             }
-            default = ngettext('expected %s argument',
-                               'expected %s arguments',
-                               action.nargs) % action.nargs
+            default = _('expected %s argument(s)') % action.nargs
             msg = nargs_errors.get(action.nargs, default)
             raise ArgumentError(action, msg)
 
@@ -2096,9 +2106,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         if len(option_tuples) > 1:
             options = ', '.join([option_string
                 for action, option_string, explicit_arg in option_tuples])
-            args = {'option': arg_string, 'matches': options}
-            msg = _('ambiguous option: %(option)s could match %(matches)s')
-            self.error(msg % args)
+            tup = arg_string, options
+            self.error(_('ambiguous option: %s could match %s') % tup)
 
         # if exactly one action matched, this segmentation is good,
         # so return the parsed action
@@ -2220,7 +2229,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
                 value = action.const
             else:
                 value = action.default
-            if isinstance(value, str):
+            if isinstance(value, basestring):
                 value = self._get_value(action, value)
                 self._check_value(action, value)
 
@@ -2277,9 +2286,8 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         # TypeErrors or ValueErrors also indicate errors
         except (TypeError, ValueError):
             name = getattr(action.type, '__name__', repr(action.type))
-            args = {'type': name, 'value': arg_string}
-            msg = _('invalid %(type)s value: %(value)r')
-            raise ArgumentError(action, msg % args)
+            msg = _('invalid %s value: %r')
+            raise ArgumentError(action, msg % (name, arg_string))
 
         # return the converted value
         return result
@@ -2287,10 +2295,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
     def _check_value(self, action, value):
         # converted value must be one of the choices (if specified)
         if action.choices is not None and value not in action.choices:
-            args = {'value': value,
-                    'choices': ', '.join(map(repr, action.choices))}
-            msg = _('invalid choice: %(value)r (choose from %(choices)s)')
-            raise ArgumentError(action, msg % args)
+            tup = value, ', '.join(map(repr, action.choices))
+            msg = _('invalid choice: %r (choose from %s)') % tup
+            raise ArgumentError(action, msg)
 
     # =======================
     # Help-formatting methods
@@ -2382,5 +2389,4 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
         should either exit or raise an exception.
         """
         self.print_usage(_sys.stderr)
-        args = {'prog': self.prog, 'message': message}
-        self.exit(2, _('%(prog)s: error: %(message)s\n') % args)
+        self.exit(2, _('%s: error: %s\n') % (self.prog, message))

lib/backports/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

lib/backports/functools_lru_cache.py

@@ -0,0 +1,196 @@
+from __future__ import absolute_import
+
+import functools
+from collections import namedtuple
+from threading import RLock
+
+_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
+
+
+@functools.wraps(functools.update_wrapper)
+def update_wrapper(
+    wrapper,
+    wrapped,
+    assigned=functools.WRAPPER_ASSIGNMENTS,
+    updated=functools.WRAPPER_UPDATES,
+):
+    """
+    Patch two bugs in functools.update_wrapper.
+    """
+    # workaround for http://bugs.python.org/issue3445
+    assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr))
+    wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated)
+    # workaround for https://bugs.python.org/issue17482
+    wrapper.__wrapped__ = wrapped
+    return wrapper
+
+
+class _HashedSeq(list):
+    __slots__ = 'hashvalue'
+
+    def __init__(self, tup, hash=hash):
+        self[:] = tup
+        self.hashvalue = hash(tup)
+
+    def __hash__(self):
+        return self.hashvalue
+
+
+def _make_key(
+    args,
+    kwds,
+    typed,
+    kwd_mark=(object(),),
+    fasttypes=set([int, str, frozenset, type(None)]),
+    sorted=sorted,
+    tuple=tuple,
+    type=type,
+    len=len,
+):
+    'Make a cache key from optionally typed positional and keyword arguments'
+    key = args
+    if kwds:
+        sorted_items = sorted(kwds.items())
+        key += kwd_mark
+        for item in sorted_items:
+            key += item
+    if typed:
+        key += tuple(type(v) for v in args)
+        if kwds:
+            key += tuple(type(v) for k, v in sorted_items)
+    elif len(key) == 1 and type(key[0]) in fasttypes:
+        return key[0]
+    return _HashedSeq(key)
+
+
+def lru_cache(maxsize=100, typed=False):
+    """Least-recently-used cache decorator.
+
+    If *maxsize* is set to None, the LRU features are disabled and the cache
+    can grow without bound.
+
+    If *typed* is True, arguments of different types will be cached separately.
+    For example, f(3.0) and f(3) will be treated as distinct calls with
+    distinct results.
+
+    Arguments to the cached function must be hashable.
+
+    View the cache statistics named tuple (hits, misses, maxsize, currsize) with
+    f.cache_info().  Clear the cache and statistics with f.cache_clear().
+    Access the underlying function with f.__wrapped__.
+
+    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
+
+    """
+
+    # Users should only access the lru_cache through its public API:
+    #       cache_info, cache_clear, and f.__wrapped__
+    # The internals of the lru_cache are encapsulated for thread safety and
+    # to allow the implementation to change (including a possible C version).
+
+    def decorating_function(user_function):
+
+        cache = dict()
+        stats = [0, 0]  # make statistics updateable non-locally
+        HITS, MISSES = 0, 1  # names for the stats fields
+        make_key = _make_key
+        cache_get = cache.get  # bound method to lookup key or return None
+        _len = len  # localize the global len() function
+        lock = RLock()  # because linkedlist updates aren't threadsafe
+        root = []  # root of the circular doubly linked list
+        root[:] = [root, root, None, None]  # initialize by pointing to self
+        nonlocal_root = [root]  # make updateable non-locally
+        PREV, NEXT, KEY, RESULT = 0, 1, 2, 3  # names for the link fields
+
+        if maxsize == 0:
+
+            def wrapper(*args, **kwds):
+                # no caching, just do a statistics update after a successful call
+                result = user_function(*args, **kwds)
+                stats[MISSES] += 1
+                return result
+
+        elif maxsize is None:
+
+            def wrapper(*args, **kwds):
+                # simple caching without ordering or size limit
+                key = make_key(args, kwds, typed)
+                result = cache_get(
+                    key, root
+                )  # root used here as a unique not-found sentinel
+                if result is not root:
+                    stats[HITS] += 1
+                    return result
+                result = user_function(*args, **kwds)
+                cache[key] = result
+                stats[MISSES] += 1
+                return result
+
+        else:
+
+            def wrapper(*args, **kwds):
+                # size limited caching that tracks accesses by recency
+                key = make_key(args, kwds, typed) if kwds or typed else args
+                with lock:
+                    link = cache_get(key)
+                    if link is not None:
+                        # record recent use of the key by moving it
+                        # to the front of the list
+                        root, = nonlocal_root
+                        link_prev, link_next, key, result = link
+                        link_prev[NEXT] = link_next
+                        link_next[PREV] = link_prev
+                        last = root[PREV]
+                        last[NEXT] = root[PREV] = link
+                        link[PREV] = last
+                        link[NEXT] = root
+                        stats[HITS] += 1
+                        return result
+                result = user_function(*args, **kwds)
+                with lock:
+                    root, = nonlocal_root
+                    if key in cache:
+                        # getting here means that this same key was added to the
+                        # cache while the lock was released.  since the link
+                        # update is already done, we need only return the
+                        # computed result and update the count of misses.
+                        pass
+                    elif _len(cache) >= maxsize:
+                        # use the old root to store the new key and result
+                        oldroot = root
+                        oldroot[KEY] = key
+                        oldroot[RESULT] = result
+                        # empty the oldest link and make it the new root
+                        root = nonlocal_root[0] = oldroot[NEXT]
+                        oldkey = root[KEY]
+                        root[KEY] = root[RESULT] = None
+                        # now update the cache dictionary for the new links
+                        del cache[oldkey]
+                        cache[key] = oldroot
+                    else:
+                        # put result in a new link at the front of the list
+                        last = root[PREV]
+                        link = [last, root, key, result]
+                        last[NEXT] = root[PREV] = cache[key] = link
+                    stats[MISSES] += 1
+                return result
+
+        def cache_info():
+            """Report cache statistics"""
+            with lock:
+                return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache))
+
+        def cache_clear():
+            """Clear the cache and cache statistics"""
+            with lock:
+                cache.clear()
+                root = nonlocal_root[0]
+                root[:] = [root, root, None, None]
+                stats[:] = [0, 0]
+
+        wrapper.__wrapped__ = user_function
+        wrapper.cache_info = cache_info
+        wrapper.cache_clear = cache_clear
+        return update_wrapper(wrapper, user_function)
+
+    return decorating_function

lib/bs4/__init__.py

@@ -5,26 +5,30 @@ http://www.crummy.com/software/BeautifulSoup/
 
 Beautiful Soup uses a pluggable XML or HTML parser to parse a
 (possibly invalid) document into a tree representation. Beautiful Soup
-provides provides methods and Pythonic idioms that make it easy to
-navigate, search, and modify the parse tree.
+provides methods and Pythonic idioms that make it easy to navigate,
+search, and modify the parse tree.
 
-Beautiful Soup works with Python 2.6 and up. It works better if lxml
+Beautiful Soup works with Python 2.7 and up. It works better if lxml
 and/or html5lib is installed.
 
 For more than you ever wanted to know about Beautiful Soup, see the
 documentation:
 http://www.crummy.com/software/BeautifulSoup/bs4/doc/
+
 """
 
 __author__ = "Leonard Richardson (leonardr@segfault.org)"
-__version__ = "4.3.2"
-__copyright__ = "Copyright (c) 2004-2013 Leonard Richardson"
+__version__ = "4.8.1"
+__copyright__ = "Copyright (c) 2004-2019 Leonard Richardson"
+# Use of this source code is governed by the MIT license.
 __license__ = "MIT"
 
 __all__ = ['BeautifulSoup']
 
 import os
 import re
+import sys
+import traceback
 import warnings
 
 from .builder import builder_registry, ParserRejectedMarkup
@@ -45,7 +49,7 @@ from .element import (
 
 # The very first thing we do is give a useful error if someone is
 # running this code under Python 3 without converting it.
-syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
+'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'!='You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
 
 class BeautifulSoup(Tag):
     """
@@ -59,7 +63,7 @@ class BeautifulSoup(Tag):
       handle_starttag(name, attrs) # See note about return value
       handle_endtag(name)
       handle_data(data) # Appends to the current data node
-      endData(containerClass=NavigableString) # Ends the current data node
+      endData(containerClass) # Ends the current data node
 
     No matter how complicated the underlying parser is, you should be
     able to build a tree using 'start tag' events, 'end tag' events,
@@ -69,21 +73,70 @@ class BeautifulSoup(Tag):
     like HTML's <br> tag), call handle_starttag and then
     handle_endtag.
     """
-    ROOT_TAG_NAME = u'[document]'
+    ROOT_TAG_NAME = '[document]'
 
     # If the end-user gives no indication which tree builder they
     # want, look for one with these features.
     DEFAULT_BUILDER_FEATURES = ['html', 'fast']
-
+   
     ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
 
+    NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features=\"%(parser)s\"' to the BeautifulSoup constructor.\n"
+
     def __init__(self, markup="", features=None, builder=None,
-                 parse_only=None, from_encoding=None, **kwargs):
-        """The Soup object is initialized as the 'root tag', and the
-        provided markup (which can be a string or a file-like object)
-        is fed into the underlying parser."""
+                 parse_only=None, from_encoding=None, exclude_encodings=None,
+                 element_classes=None, **kwargs):
+        """Constructor.
+
+        :param markup: A string or a file-like object representing
+        markup to be parsed.
+
+        :param features: Desirable features of the parser to be used. This
+        may be the name of a specific parser ("lxml", "lxml-xml",
+        "html.parser", or "html5lib") or it may be the type of markup
+        to be used ("html", "html5", "xml"). It's recommended that you
+        name a specific parser, so that Beautiful Soup gives you the
+        same results across platforms and virtual environments.
+
+        :param builder: A TreeBuilder subclass to instantiate (or
+        instance to use) instead of looking one up based on
+        `features`. You only need to use this if you've implemented a
+        custom TreeBuilder.
+
+        :param parse_only: A SoupStrainer. Only parts of the document
+        matching the SoupStrainer will be considered. This is useful
+        when parsing part of a document that would otherwise be too
+        large to fit into memory.
+
+        :param from_encoding: A string indicating the encoding of the
+        document to be parsed. Pass this in if Beautiful Soup is
+        guessing wrongly about the document's encoding.
+
+        :param exclude_encodings: A list of strings indicating
+        encodings known to be wrong. Pass this in if you don't know
+        the document's encoding but you know Beautiful Soup's guess is
+        wrong.
+
+        :param element_classes: A dictionary mapping BeautifulSoup
+        classes like Tag and NavigableString to other classes you'd
+        like to be instantiated instead as the parse tree is
+        built. This is useful for using subclasses to modify the
+        default behavior of Tag or NavigableString.
+
+        :param kwargs: For backwards compatibility purposes, the
+        constructor accepts certain keyword arguments used in
+        Beautiful Soup 3. None of these arguments do anything in
+        Beautiful Soup 4; they will result in a warning and then be ignored.
+
+        Apart from this, any keyword arguments passed into the BeautifulSoup
+        constructor are propagated to the TreeBuilder constructor. This
+        makes it possible to configure a TreeBuilder beyond saying
+        which one to use.
+
+        """
 
         if 'convertEntities' in kwargs:
+            del kwargs['convertEntities']
             warnings.warn(
                 "BS4 does not respect the convertEntities argument to the "
                 "BeautifulSoup constructor. Entities are always converted "
@@ -114,9 +167,9 @@ class BeautifulSoup(Tag):
             del kwargs['isHTML']
             warnings.warn(
                 "BS4 does not respect the isHTML argument to the "
-                "BeautifulSoup constructor. You can pass in features='html' "
-                "or features='xml' to get a builder capable of handling "
-                "one or the other.")
+                "BeautifulSoup constructor. Suggest you use "
+                "features='lxml' for HTML and features='lxml-xml' for "
+                "XML.")
 
         def deprecated_argument(old_name, new_name):
             if old_name in kwargs:
@@ -134,13 +187,24 @@ class BeautifulSoup(Tag):
         from_encoding = from_encoding or deprecated_argument(
             "fromEncoding", "from_encoding")
 
-        if len(kwargs) > 0:
-            arg = kwargs.keys().pop()
-            raise TypeError(
-                "__init__() got an unexpected keyword argument '%s'" % arg)
+        if from_encoding and isinstance(markup, str):
+            warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.")
+            from_encoding = None
 
-        if builder is None:
-            if isinstance(features, basestring):
+        self.element_classes = element_classes or dict()
+
+        # We need this information to track whether or not the builder
+        # was specified well enough that we can omit the 'you need to
+        # specify a parser' warning.
+        original_builder = builder
+        original_features = features
+            
+        if isinstance(builder, type):
+            # A builder class was passed in; it needs to be instantiated.
+            builder_class = builder
+            builder = None
+        elif builder is None:
+            if isinstance(features, str):
                 features = [features]
             if features is None or len(features) == 0:
                 features = self.DEFAULT_BUILDER_FEATURES
@@ -150,21 +214,73 @@ class BeautifulSoup(Tag):
                     "Couldn't find a tree builder with the features you "
                     "requested: %s. Do you need to install a parser library?"
                     % ",".join(features))
-            builder = builder_class()
+
+        # At this point either we have a TreeBuilder instance in
+        # builder, or we have a builder_class that we can instantiate
+        # with the remaining **kwargs.
+        if builder is None:
+            builder = builder_class(**kwargs)
+            if not original_builder and not (
+                    original_features == builder.NAME or
+                    original_features in builder.ALTERNATE_NAMES
+            ):
+                if builder.is_xml:
+                    markup_type = "XML"
+                else:
+                    markup_type = "HTML"
+
+                # This code adapted from warnings.py so that we get the same line
+                # of code as our warnings.warn() call gets, even if the answer is wrong
+                # (as it may be in a multithreading situation).
+                caller = None
+                try:
+                    caller = sys._getframe(1)
+                except ValueError:
+                    pass
+                if caller:
+                    globals = caller.f_globals
+                    line_number = caller.f_lineno
+                else:
+                    globals = sys.__dict__
+                    line_number= 1                    
+                filename = globals.get('__file__')
+                if filename:
+                    fnl = filename.lower()
+                    if fnl.endswith((".pyc", ".pyo")):
+                        filename = filename[:-1]
+                if filename:
+                    # If there is no filename at all, the user is most likely in a REPL,
+                    # and the warning is not necessary.
+                    values = dict(
+                        filename=filename,
+                        line_number=line_number,
+                        parser=builder.NAME,
+                        markup_type=markup_type
+                    )
+                    warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % values, stacklevel=2)
+        else:
+            if kwargs:
+                warnings.warn("Keyword arguments to the BeautifulSoup constructor will be ignored. These would normally be passed into the TreeBuilder constructor, but a TreeBuilder instance was passed in as `builder`.")
+                    
         self.builder = builder
         self.is_xml = builder.is_xml
-        self.builder.soup = self
-
+        self.known_xml = self.is_xml
+        self._namespaces = dict()
         self.parse_only = parse_only
 
+        self.builder.initialize_soup(self)
+
         if hasattr(markup, 'read'):        # It's a file-type object.
             markup = markup.read()
-        elif len(markup) <= 256:
+        elif len(markup) <= 256 and (
+                (isinstance(markup, bytes) and not b'<' in markup)
+                or (isinstance(markup, str) and not '<' in markup)
+        ):
             # Print out warnings for a couple beginner problems
             # involving passing non-markup to Beautiful Soup.
             # Beautiful Soup will still parse the input as markup,
             # just in case that's what the user really wants.
-            if (isinstance(markup, unicode)
+            if (isinstance(markup, str)
                 and not os.path.supports_unicode_filenames):
                 possible_filename = markup.encode("utf8")
             else:
@@ -172,37 +288,93 @@ class BeautifulSoup(Tag):
             is_file = False
             try:
                 is_file = os.path.exists(possible_filename)
-            except Exception, e:
+            except Exception as e:
                 # This is almost certainly a problem involving
                 # characters not valid in filenames on this
                 # system. Just let it go.
                 pass
             if is_file:
+                if isinstance(markup, str):
+                    markup = markup.encode("utf8")
                 warnings.warn(
-                    '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup)
-            if markup[:5] == "http:" or markup[:6] == "https:":
-                # TODO: This is ugly but I couldn't get it to work in
-                # Python 3 otherwise.
-                if ((isinstance(markup, bytes) and not b' ' in markup)
-                    or (isinstance(markup, unicode) and not u' ' in markup)):
-                    warnings.warn(
-                        '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)
+                    '"%s" looks like a filename, not markup. You should'
+                    ' probably open this file and pass the filehandle into'
+                    ' Beautiful Soup.' % markup)
+            self._check_markup_is_url(markup)
 
+        rejections = []
+        success = False
         for (self.markup, self.original_encoding, self.declared_html_encoding,
          self.contains_replacement_characters) in (
-            self.builder.prepare_markup(markup, from_encoding)):
+             self.builder.prepare_markup(
+                 markup, from_encoding, exclude_encodings=exclude_encodings)):
             self.reset()
             try:
                 self._feed()
+                success = True
                 break
-            except ParserRejectedMarkup:
+            except ParserRejectedMarkup as e:
+                rejections.append(e)
                 pass
 
+        if not success:
+            other_exceptions = [str(e) for e in rejections]
+            raise ParserRejectedMarkup(
+                "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.\n\nOriginal exception(s) from parser:\n " + "\n ".join(other_exceptions)
+            )
+
         # Clear out the markup and remove the builder's circular
         # reference to this object.
         self.markup = None
         self.builder.soup = None
 
+    def __copy__(self):
+        copy = type(self)(
+            self.encode('utf-8'), builder=self.builder, from_encoding='utf-8'
+        )
+
+        # Although we encoded the tree to UTF-8, that may not have
+        # been the encoding of the original markup. Set the copy's
+        # .original_encoding to reflect the original object's
+        # .original_encoding.
+        copy.original_encoding = self.original_encoding
+        return copy
+
+    def __getstate__(self):
+        # Frequently a tree builder can't be pickled.
+        d = dict(self.__dict__)
+        if 'builder' in d and not self.builder.picklable:
+            d['builder'] = None
+        return d
+
+    @staticmethod
+    def _check_markup_is_url(markup):
+        """ 
+        Check if markup looks like it's actually a url and raise a warning 
+        if so. Markup can be unicode or str (py2) / bytes (py3).
+        """
+        if isinstance(markup, bytes):
+            space = b' '
+            cant_start_with = (b"http:", b"https:")
+        elif isinstance(markup, str):
+            space = ' '
+            cant_start_with = ("http:", "https:")
+        else:
+            return
+
+        if any(markup.startswith(prefix) for prefix in cant_start_with):
+            if not space in markup:
+                if isinstance(markup, bytes):
+                    decoded_markup = markup.decode('utf-8', 'replace')
+                else:
+                    decoded_markup = markup
+                warnings.warn(
+                    '"%s" looks like a URL. Beautiful Soup is not an'
+                    ' HTTP client. You should probably use an HTTP client like'
+                    ' requests to get the document behind the URL, and feed'
+                    ' that document to Beautiful Soup.' % decoded_markup
+                )
+
     def _feed(self):
         # Convert the document to Unicode.
         self.builder.reset()
@@ -223,15 +395,21 @@ class BeautifulSoup(Tag):
         self.preserve_whitespace_tag_stack = []
         self.pushTag(self)
 
-    def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
+    def new_tag(self, name, namespace=None, nsprefix=None, attrs={},
+                sourceline=None, sourcepos=None, **kwattrs):
         """Create a new tag associated with this soup."""
-        return Tag(None, self.builder, name, namespace, nsprefix, attrs)
+        kwattrs.update(attrs)
+        return self.element_classes.get(Tag, Tag)(
+            None, self.builder, name, namespace, nsprefix, kwattrs,
+            sourceline=sourceline, sourcepos=sourcepos
+        )
 
-    def new_string(self, s, subclass=NavigableString):
+    def new_string(self, s, subclass=None):
         """Create a new NavigableString associated with this soup."""
-        navigable = subclass(s)
-        navigable.setup()
-        return navigable
+        subclass = subclass or self.element_classes.get(
+            NavigableString, NavigableString
+        )
+        return subclass(s)
 
     def insert_before(self, successor):
         raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
@@ -250,16 +428,26 @@ class BeautifulSoup(Tag):
 
     def pushTag(self, tag):
         #print "Push", tag.name
-        if self.currentTag:
+        if self.currentTag is not None:
             self.currentTag.contents.append(tag)
         self.tagStack.append(tag)
         self.currentTag = self.tagStack[-1]
         if tag.name in self.builder.preserve_whitespace_tags:
             self.preserve_whitespace_tag_stack.append(tag)
 
-    def endData(self, containerClass=NavigableString):
+    def endData(self, containerClass=None):
+
+        # Default container is NavigableString.
+        containerClass = containerClass or NavigableString
+
+        # The user may want us to instantiate some alias for the
+        # container class.
+        containerClass = self.element_classes.get(
+            containerClass, containerClass
+        )
+        
         if self.current_data:
-            current_data = u''.join(self.current_data)
+            current_data = ''.join(self.current_data)
             # If whitespace is not preserved, and this string contains
             # nothing but ASCII spaces, replace it with a single space
             # or newline.
@@ -289,15 +477,72 @@ class BeautifulSoup(Tag):
 
     def object_was_parsed(self, o, parent=None, most_recent_element=None):
         """Add an object to the parse tree."""
-        parent = parent or self.currentTag
-        most_recent_element = most_recent_element or self._most_recent_element
-        o.setup(parent, most_recent_element)
-
+        if parent is None:
+            parent = self.currentTag
         if most_recent_element is not None:
-            most_recent_element.next_element = o
+            previous_element = most_recent_element
+        else:
+            previous_element = self._most_recent_element
+
+        next_element = previous_sibling = next_sibling = None
+        if isinstance(o, Tag):
+            next_element = o.next_element
+            next_sibling = o.next_sibling
+            previous_sibling = o.previous_sibling
+            if previous_element is None:
+                previous_element = o.previous_element
+
+        fix = parent.next_element is not None
+
+        o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)
+
         self._most_recent_element = o
         parent.contents.append(o)
 
+        # Check if we are inserting into an already parsed node.
+        if fix:
+            self._linkage_fixer(parent)
+
+    def _linkage_fixer(self, el):
+        """Make sure linkage of this fragment is sound."""
+
+        first = el.contents[0]
+        child = el.contents[-1]
+        descendant = child
+
+        if child is first and el.parent is not None:
+            # Parent should be linked to first child
+            el.next_element = child
+            # We are no longer linked to whatever this element is
+            prev_el = child.previous_element
+            if prev_el is not None and prev_el is not el:
+                prev_el.next_element = None
+            # First child should be linked to the parent, and no previous siblings.
+            child.previous_element = el
+            child.previous_sibling = None
+
+        # We have no sibling as we've been appended as the last.
+        child.next_sibling = None
+
+        # This index is a tag, dig deeper for a "last descendant"
+        if isinstance(child, Tag) and child.contents:
+            descendant = child._last_descendant(False)
+
+        # As the final step, link last descendant. It should be linked
+        # to the parent's next sibling (if found), else walk up the chain
+        # and find a parent with a sibling. It should have no next sibling.
+        descendant.next_element = None
+        descendant.next_sibling = None
+        target = el
+        while True:
+            if target is None:
+                break
+            elif target.next_sibling is not None:
+                descendant.next_element = target.next_sibling
+                target.next_sibling.previous_element = child
+                break
+            target = target.parent
+
     def _popToTag(self, name, nsprefix=None, inclusivePop=True):
         """Pops the tag stack up to and including the most recent
         instance of the given tag. If inclusivePop is false, pops the tag
@@ -321,11 +566,12 @@ class BeautifulSoup(Tag):
 
         return most_recently_popped
 
-    def handle_starttag(self, name, namespace, nsprefix, attrs):
+    def handle_starttag(self, name, namespace, nsprefix, attrs, sourceline=None,
+                        sourcepos=None):
         """Push a start tag on to the stack.
 
         If this method returns None, the tag was rejected by the
-        SoupStrainer. You should proceed as if the tag had not occured
+        SoupStrainer. You should proceed as if the tag had not occurred
         in the document. For instance, if this was a self-closing tag,
         don't call handle_endtag.
         """
@@ -338,11 +584,14 @@ class BeautifulSoup(Tag):
                  or not self.parse_only.search_tag(name, attrs))):
             return None
 
-        tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
-                  self.currentTag, self._most_recent_element)
+        tag = self.element_classes.get(Tag, Tag)(
+            self, self.builder, name, namespace, nsprefix, attrs,
+            self.currentTag, self._most_recent_element,
+            sourceline=sourceline, sourcepos=sourcepos
+        )
         if tag is None:
             return tag
-        if self._most_recent_element:
+        if self._most_recent_element is not None:
             self._most_recent_element.next_element = tag
         self._most_recent_element = tag
         self.pushTag(tag)
@@ -367,9 +616,9 @@ class BeautifulSoup(Tag):
             encoding_part = ''
             if eventual_encoding != None:
                 encoding_part = ' encoding="%s"' % eventual_encoding
-            prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
+            prefix = '<?xml version="1.0"%s?>\n' % encoding_part
         else:
-            prefix = u''
+            prefix = ''
         if not pretty_print:
             indent_level = None
         else:
@@ -403,4 +652,4 @@ class FeatureNotFound(ValueError):
 if __name__ == '__main__':
     import sys
     soup = BeautifulSoup(sys.stdin)
-    print soup.prettify()
+    print(soup.prettify())

lib/bs4/builder/__init__.py

@@ -1,10 +1,13 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 from collections import defaultdict
 import itertools
 import sys
 from bs4.element import (
     CharsetMetaAttributeValue,
     ContentMetaAttributeValue,
-    whitespace_re
+    nonwhitespace_re
     )
 
 __all__ = [
@@ -80,21 +83,70 @@ builder_registry = TreeBuilderRegistry()
 class TreeBuilder(object):
     """Turn a document into a Beautiful Soup object tree."""
 
+    NAME = "[Unknown tree builder]"
+    ALTERNATE_NAMES = []
     features = []
 
     is_xml = False
-    preserve_whitespace_tags = set()
+    picklable = False
     empty_element_tags = None # A tag will be considered an empty-element
                               # tag when and only when it has no contents.
-
+    
     # A value for these tag/attribute combinations is a space- or
     # comma-separated list of CDATA, rather than a single CDATA.
-    cdata_list_attributes = {}
+    DEFAULT_CDATA_LIST_ATTRIBUTES = {}
 
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set()
+    
+    USE_DEFAULT = object()
 
-    def __init__(self):
+    # Most parsers don't keep track of line numbers.
+    TRACKS_LINE_NUMBERS = False
+    
+    def __init__(self, multi_valued_attributes=USE_DEFAULT,
+                 preserve_whitespace_tags=USE_DEFAULT,
+                 store_line_numbers=USE_DEFAULT):
+        """Constructor.
+
+        :param multi_valued_attributes: If this is set to None, the
+        TreeBuilder will not turn any values for attributes like
+        'class' into lists. Setting this do a dictionary will
+        customize this behavior; look at DEFAULT_CDATA_LIST_ATTRIBUTES
+        for an example.
+
+        Internally, these are called "CDATA list attributes", but that
+        probably doesn't make sense to an end-user, so the argument name
+        is `multi_valued_attributes`.
+
+        :param preserve_whitespace_tags: A list of tags to treat
+        the way <pre> tags are treated in HTML. Tags in this list
+        will have 
+
+        :param store_line_numbers: If the parser keeps track of the
+        line numbers and positions of the original markup, that
+        information will, by default, be stored in each corresponding
+        `Tag` object. You can turn this off by passing
+        store_line_numbers=False. If the parser you're using doesn't 
+        keep track of this information, then setting store_line_numbers=True
+        will do nothing.
+        """
         self.soup = None
-
+        if multi_valued_attributes is self.USE_DEFAULT:
+            multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES
+        self.cdata_list_attributes = multi_valued_attributes
+        if preserve_whitespace_tags is self.USE_DEFAULT:
+            preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS
+        self.preserve_whitespace_tags = preserve_whitespace_tags
+        if store_line_numbers == self.USE_DEFAULT:
+            store_line_numbers = self.TRACKS_LINE_NUMBERS
+        self.store_line_numbers = store_line_numbers
+        
+    def initialize_soup(self, soup):
+        """The BeautifulSoup object has been initialized and is now
+        being associated with the TreeBuilder.
+        """
+        self.soup = soup
+        
     def reset(self):
         pass
 
@@ -118,13 +170,13 @@ class TreeBuilder(object):
         if self.empty_element_tags is None:
             return True
         return tag_name in self.empty_element_tags
-
+    
     def feed(self, markup):
         raise NotImplementedError()
 
     def prepare_markup(self, markup, user_specified_encoding=None,
-                       document_declared_encoding=None):
-        return markup, None, None, False
+                       document_declared_encoding=None, exclude_encodings=None):
+        yield markup, None, None, False
 
     def test_fragment_to_document(self, fragment):
         """Wrap an HTML fragment to make it look like a document.
@@ -153,14 +205,14 @@ class TreeBuilder(object):
             universal = self.cdata_list_attributes.get('*', [])
             tag_specific = self.cdata_list_attributes.get(
                 tag_name.lower(), None)
-            for attr in attrs.keys():
+            for attr in list(attrs.keys()):
                 if attr in universal or (tag_specific and attr in tag_specific):
                     # We have a "class"-type attribute whose string
                     # value is a whitespace-separated list of
                     # values. Split it into a list.
                     value = attrs[attr]
-                    if isinstance(value, basestring):
-                        values = whitespace_re.split(value)
+                    if isinstance(value, str):
+                        values = nonwhitespace_re.findall(value)
                     else:
                         # html5lib sometimes calls setAttributes twice
                         # for the same tag when rearranging the parse
@@ -224,10 +276,20 @@ class HTMLTreeBuilder(TreeBuilder):
     Such as which tags are empty-element tags.
     """
 
-    preserve_whitespace_tags = set(['pre', 'textarea'])
-    empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
-                              'spacer', 'link', 'frame', 'base'])
+    empty_element_tags = set([
+        # These are from HTML5.
+        'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
+        
+        # These are from earlier versions of HTML and are removed in HTML5.
+        'basefont', 'bgsound', 'command', 'frame', 'image', 'isindex', 'nextid', 'spacer'
+    ])
 
+    # The HTML standard defines these as block-level elements. Beautiful
+    # Soup does not treat these elements differently from other elements,
+    # but it may do so eventually, and this information is available if
+    # you need to use it.
+    block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"])
+    
     # The HTML standard defines these attributes as containing a
     # space-separated list of values, not a single value. That is,
     # class="foo bar" means that the 'class' attribute has two values,
@@ -235,7 +297,7 @@ class HTMLTreeBuilder(TreeBuilder):
     # encounter one of these attributes, we will parse its value into
     # a list of values if possible. Upon output, the list will be
     # converted back into a string.
-    cdata_list_attributes = {
+    DEFAULT_CDATA_LIST_ATTRIBUTES = {
         "*" : ['class', 'accesskey', 'dropzone'],
         "a" : ['rel', 'rev'],
         "link" :  ['rel', 'rev'],
@@ -252,6 +314,8 @@ class HTMLTreeBuilder(TreeBuilder):
         "output" : ["for"],
         }
 
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
+    
     def set_up_substitutions(self, tag):
         # We are only interested in <meta> tags
         if tag.name != 'meta':
@@ -299,8 +363,15 @@ def register_treebuilders_from(module):
             this_module.builder_registry.register(obj)
 
 class ParserRejectedMarkup(Exception):
-    pass
-
+    def __init__(self, message_or_exception):
+        """Explain why the parser rejected the given markup, either
+        with a textual explanation or another exception.
+        """
+        if isinstance(message_or_exception, Exception):
+            e = message_or_exception
+            message_or_exception = "%s: %s" % (e.__class__.__name__, str(e))
+        super(ParserRejectedMarkup, self).__init__(message_or_exception)
+            
 # Builders are registered in reverse order of priority, so that custom
 # builder registrations will take precedence. In general, we want lxml
 # to take precedence over html5lib, because it's faster. And we only

lib/bs4/builder/_html5lib.py

@@ -1,17 +1,27 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'HTML5TreeBuilder',
     ]
 
 import warnings
+import re
 from bs4.builder import (
     PERMISSIVE,
     HTML,
     HTML_5,
     HTMLTreeBuilder,
     )
-from bs4.element import NamespacedAttribute
+from bs4.element import (
+    NamespacedAttribute,
+    nonwhitespace_re,
+)
 import html5lib
-from html5lib.constants import namespaces
+from html5lib.constants import (
+    namespaces,
+    prefixes,
+    )
 from bs4.element import (
     Comment,
     Doctype,
@@ -19,14 +29,36 @@ from bs4.element import (
     Tag,
     )
 
+try:
+    # Pre-0.99999999
+    from html5lib.treebuilders import _base as treebuilder_base
+    new_html5lib = False
+except ImportError as e:
+    # 0.99999999 and up
+    from html5lib.treebuilders import base as treebuilder_base
+    new_html5lib = True
+
 class HTML5TreeBuilder(HTMLTreeBuilder):
     """Use html5lib to build a tree."""
 
-    features = ['html5lib', PERMISSIVE, HTML_5, HTML]
+    NAME = "html5lib"
 
-    def prepare_markup(self, markup, user_specified_encoding):
+    features = [NAME, PERMISSIVE, HTML_5, HTML]
+
+    # html5lib can tell us which line number and position in the
+    # original file is the source of an element.
+    TRACKS_LINE_NUMBERS = True
+    
+    def prepare_markup(self, markup, user_specified_encoding,
+                       document_declared_encoding=None, exclude_encodings=None):
         # Store the user-specified encoding for use later on.
         self.user_specified_encoding = user_specified_encoding
+
+        # document_declared_encoding and exclude_encodings aren't used
+        # ATM because the html5lib TreeBuilder doesn't use
+        # UnicodeDammit.
+        if exclude_encodings:
+            warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
         yield (markup, None, None, False)
 
     # These methods are defined by Beautiful Soup.
@@ -34,32 +66,63 @@ class HTML5TreeBuilder(HTMLTreeBuilder):
         if self.soup.parse_only is not None:
             warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
         parser = html5lib.HTMLParser(tree=self.create_treebuilder)
-        doc = parser.parse(markup, encoding=self.user_specified_encoding)
-
+        self.underlying_builder.parser = parser
+        extra_kwargs = dict()
+        if not isinstance(markup, str):
+            if new_html5lib:
+                extra_kwargs['override_encoding'] = self.user_specified_encoding
+            else:
+                extra_kwargs['encoding'] = self.user_specified_encoding
+        doc = parser.parse(markup, **extra_kwargs)
+        
         # Set the character encoding detected by the tokenizer.
-        if isinstance(markup, unicode):
+        if isinstance(markup, str):
             # We need to special-case this because html5lib sets
             # charEncoding to UTF-8 if it gets Unicode input.
             doc.original_encoding = None
         else:
-            doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
-
+            original_encoding = parser.tokenizer.stream.charEncoding[0]
+            if not isinstance(original_encoding, str):
+                # In 0.99999999 and up, the encoding is an html5lib
+                # Encoding object. We want to use a string for compatibility
+                # with other tree builders.
+                original_encoding = original_encoding.name
+            doc.original_encoding = original_encoding
+        self.underlying_builder.parser = None
+            
     def create_treebuilder(self, namespaceHTMLElements):
         self.underlying_builder = TreeBuilderForHtml5lib(
-            self.soup, namespaceHTMLElements)
+            namespaceHTMLElements, self.soup,
+            store_line_numbers=self.store_line_numbers
+        )
         return self.underlying_builder
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""
-        return u'<html><head></head><body>%s</body></html>' % fragment
+        return '<html><head></head><body>%s</body></html>' % fragment
 
 
-class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
-
-    def __init__(self, soup, namespaceHTMLElements):
-        self.soup = soup
+class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
+    
+    def __init__(self, namespaceHTMLElements, soup=None,
+                 store_line_numbers=True, **kwargs):
+        if soup:
+            self.soup = soup
+        else:
+            from bs4 import BeautifulSoup
+            # TODO: Why is the parser 'html.parser' here? To avoid an
+            # infinite loop?
+            self.soup = BeautifulSoup(
+                "", "html.parser", store_line_numbers=store_line_numbers,
+                **kwargs
+            )
         super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
 
+        # This will be set later to an html5lib.html5parser.HTMLParser
+        # object, which we can use to track the current line number.
+        self.parser = None
+        self.store_line_numbers = store_line_numbers
+        
     def documentClass(self):
         self.soup.reset()
         return Element(self.soup, self.soup, None)
@@ -73,14 +136,26 @@ class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
         self.soup.object_was_parsed(doctype)
 
     def elementClass(self, name, namespace):
-        tag = self.soup.new_tag(name, namespace)
+        kwargs = {}
+        if self.parser and self.store_line_numbers:
+            # This represents the point immediately after the end of the
+            # tag. We don't know when the tag started, but we do know
+            # where it ended -- the character just before this one.
+            sourceline, sourcepos = self.parser.tokenizer.stream.position()
+            kwargs['sourceline'] = sourceline
+            kwargs['sourcepos'] = sourcepos-1
+        tag = self.soup.new_tag(name, namespace, **kwargs)
+
         return Element(tag, self.soup, namespace)
 
     def commentClass(self, data):
         return TextNode(Comment(data), self.soup)
 
     def fragmentClass(self):
-        self.soup = BeautifulSoup("")
+        from bs4 import BeautifulSoup
+        # TODO: Why is the parser 'html.parser' here? To avoid an
+        # infinite loop?
+        self.soup = BeautifulSoup("", "html.parser")
         self.soup.name = "[document_fragment]"
         return Element(self.soup, self.soup, None)
 
@@ -92,7 +167,57 @@ class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
         return self.soup
 
     def getFragment(self):
-        return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
+        return treebuilder_base.TreeBuilder.getFragment(self).element
+
+    def testSerializer(self, element):
+        from bs4 import BeautifulSoup
+        rv = []
+        doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$')
+
+        def serializeElement(element, indent=0):
+            if isinstance(element, BeautifulSoup):
+                pass
+            if isinstance(element, Doctype):
+                m = doctype_re.match(element)
+                if m:
+                    name = m.group(1)
+                    if m.lastindex > 1:
+                        publicId = m.group(2) or ""
+                        systemId = m.group(3) or m.group(4) or ""
+                        rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (' ' * indent, name, publicId, systemId))
+                    else:
+                        rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name))
+                else:
+                    rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
+            elif isinstance(element, Comment):
+                rv.append("|%s<!-- %s -->" % (' ' * indent, element))
+            elif isinstance(element, NavigableString):
+                rv.append("|%s\"%s\"" % (' ' * indent, element))
+            else:
+                if element.namespace:
+                    name = "%s %s" % (prefixes[element.namespace],
+                                      element.name)
+                else:
+                    name = element.name
+                rv.append("|%s<%s>" % (' ' * indent, name))
+                if element.attrs:
+                    attributes = []
+                    for name, value in list(element.attrs.items()):
+                        if isinstance(name, NamespacedAttribute):
+                            name = "%s %s" % (prefixes[name.namespace], name.name)
+                        if isinstance(value, list):
+                            value = " ".join(value)
+                        attributes.append((name, value))
+
+                    for name, value in sorted(attributes):
+                        rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
+                indent += 2
+                for child in element.children:
+                    serializeElement(child, indent)
+        serializeElement(element, 0)
+
+        return "\n".join(rv)
 
 class AttrList(object):
     def __init__(self, element):
@@ -101,7 +226,16 @@ class AttrList(object):
     def __iter__(self):
         return list(self.attrs.items()).__iter__()
     def __setitem__(self, name, value):
-        "set attr", name, value
+        # If this attribute is a multi-valued attribute for this element,
+        # turn its value into a list.
+        list_attr = self.element.cdata_list_attributes
+        if (name in list_attr['*']
+            or (self.element.name in list_attr
+                and name in list_attr[self.element.name])):
+            # A node that is being cloned may have already undergone
+            # this procedure.
+            if not isinstance(value, list):
+                value = nonwhitespace_re.findall(value)
         self.element[name] = value
     def items(self):
         return list(self.attrs.items())
@@ -115,16 +249,16 @@ class AttrList(object):
         return name in list(self.attrs.keys())
 
 
-class Element(html5lib.treebuilders._base.Node):
+class Element(treebuilder_base.Node):
     def __init__(self, element, soup, namespace):
-        html5lib.treebuilders._base.Node.__init__(self, element.name)
+        treebuilder_base.Node.__init__(self, element.name)
         self.element = element
         self.soup = soup
         self.namespace = namespace
 
     def appendChild(self, node):
         string_child = child = None
-        if isinstance(node, basestring):
+        if isinstance(node, str):
             # Some other piece of code decided to pass in a string
             # instead of creating a TextElement object to contain the
             # string.
@@ -136,13 +270,15 @@ class Element(html5lib.treebuilders._base.Node):
             child = node
         elif node.element.__class__ == NavigableString:
             string_child = child = node.element
+            node.parent = self
         else:
             child = node.element
+            node.parent = self
 
-        if not isinstance(child, basestring) and child.parent is not None:
+        if not isinstance(child, str) and child.parent is not None:
             node.element.extract()
 
-        if (string_child and self.element.contents
+        if (string_child is not None and self.element.contents
             and self.element.contents[-1].__class__ == NavigableString):
             # We are appending a string onto another string.
             # TODO This has O(n^2) performance, for input like
@@ -152,7 +288,7 @@ class Element(html5lib.treebuilders._base.Node):
             old_element.replace_with(new_element)
             self.soup._most_recent_element = new_element
         else:
-            if isinstance(node, basestring):
+            if isinstance(node, str):
                 # Create a brand new NavigableString from this string.
                 child = self.soup.new_string(node)
 
@@ -161,6 +297,12 @@ class Element(html5lib.treebuilders._base.Node):
             # immediately after the parent, if it has no children.)
             if self.element.contents:
                 most_recent_element = self.element._last_descendant(False)
+            elif self.element.next_element is not None:
+                # Something from further ahead in the parse tree is
+                # being inserted into this earlier element. This is
+                # very annoying because it means an expensive search
+                # for the last element in the tree.
+                most_recent_element = self.soup._last_descendant()
             else:
                 most_recent_element = self.element
 
@@ -169,9 +311,12 @@ class Element(html5lib.treebuilders._base.Node):
                 most_recent_element=most_recent_element)
 
     def getAttributes(self):
+        if isinstance(self.element, Comment):
+            return {}
         return AttrList(self.element)
 
     def setAttributes(self, attributes):
+
         if attributes is not None and len(attributes) > 0:
 
             converted_attributes = []
@@ -183,7 +328,7 @@ class Element(html5lib.treebuilders._base.Node):
 
             self.soup.builder._replace_cdata_list_attribute_values(
                 self.name, attributes)
-            for name, value in attributes.items():
+            for name, value in list(attributes.items()):
                 self.element[name] = value
 
             # The attributes may contain variables that need substitution.
@@ -195,11 +340,11 @@ class Element(html5lib.treebuilders._base.Node):
     attributes = property(getAttributes, setAttributes)
 
     def insertText(self, data, insertBefore=None):
+        text = TextNode(self.soup.new_string(data), self.soup)
         if insertBefore:
-            text = TextNode(self.soup.new_string(data), self.soup)
-            self.insertBefore(data, insertBefore)
+            self.insertBefore(text, insertBefore)
         else:
-            self.appendChild(data)
+            self.appendChild(text)
 
     def insertBefore(self, node, refNode):
         index = self.element.index(refNode.element)
@@ -218,6 +363,10 @@ class Element(html5lib.treebuilders._base.Node):
 
     def reparentChildren(self, new_parent):
         """Move all of this tag's children into another tag."""
+        # print "MOVE", self.element.contents
+        # print "FROM", self.element
+        # print "TO", new_parent.element
+
         element = self.element
         new_parent_element = new_parent.element
         # Determine what this tag's next_element will be once all the children
@@ -236,18 +385,35 @@ class Element(html5lib.treebuilders._base.Node):
             new_parents_last_descendant_next_element = new_parent_element.next_element
 
         to_append = element.contents
-        append_after = new_parent.element.contents
         if len(to_append) > 0:
             # Set the first child's previous_element and previous_sibling
             # to elements within the new parent
             first_child = to_append[0]
-            first_child.previous_element = new_parents_last_descendant
+            if new_parents_last_descendant is not None:
+                first_child.previous_element = new_parents_last_descendant
+            else:
+                first_child.previous_element = new_parent_element
             first_child.previous_sibling = new_parents_last_child
+            if new_parents_last_descendant is not None:
+                new_parents_last_descendant.next_element = first_child
+            else:
+                new_parent_element.next_element = first_child
+            if new_parents_last_child is not None:
+                new_parents_last_child.next_sibling = first_child
 
-            # Fix the last child's next_element and next_sibling
-            last_child = to_append[-1]
-            last_child.next_element = new_parents_last_descendant_next_element
-            last_child.next_sibling = None
+            # Find the very last element being moved. It is now the
+            # parent's last descendant. It has no .next_sibling and
+            # its .next_element is whatever the previous last
+            # descendant had.
+            last_childs_last_descendant = to_append[-1]._last_descendant(False, True)
+
+            last_childs_last_descendant.next_element = new_parents_last_descendant_next_element
+            if new_parents_last_descendant_next_element is not None:
+                # TODO: This code has no test coverage and I'm not sure
+                # how to get html5lib to go through this path, but it's
+                # just the other side of the previous line.
+                new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant
+            last_childs_last_descendant.next_sibling = None
 
         for child in to_append:
             child.parent = new_parent_element
@@ -257,6 +423,10 @@ class Element(html5lib.treebuilders._base.Node):
         element.contents = []
         element.next_element = final_next_element
 
+        # print "DONE WITH MOVE"
+        # print "FROM", self.element
+        # print "TO", new_parent_element
+
     def cloneNode(self):
         tag = self.soup.new_tag(self.element.name, self.namespace)
         node = Element(tag, self.soup, self.namespace)
@@ -268,7 +438,7 @@ class Element(html5lib.treebuilders._base.Node):
         return self.element.contents
 
     def getNameTuple(self):
-        if self.namespace is None:
+        if self.namespace == None:
             return namespaces["html"], self.name
         else:
             return self.namespace, self.name
@@ -277,7 +447,7 @@ class Element(html5lib.treebuilders._base.Node):
 
 class TextNode(Element):
     def __init__(self, element, soup):
-        html5lib.treebuilders._base.Node.__init__(self, None)
+        treebuilder_base.Node.__init__(self, None)
         self.element = element
         self.soup = soup
 

lib/bs4/builder/_htmlparser.py

@@ -1,13 +1,23 @@
+# encoding: utf-8
 """Use the HTMLParser library to parse HTML files that aren't too bad."""
 
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'HTMLParserTreeBuilder',
     ]
 
-from HTMLParser import (
-    HTMLParser,
-    HTMLParseError,
-    )
+from future.moves.html.parser import HTMLParser
+
+try:
+    from html.parser import HTMLParseError
+except ImportError as e:
+    # HTMLParseError is removed in Python 3.5. Since it can never be
+    # thrown in 3.5, we can just define our own class as a placeholder.
+    class HTMLParseError(Exception):
+        pass
+
 import sys
 import warnings
 
@@ -19,10 +29,10 @@ import warnings
 # At the end of this file, we monkeypatch HTMLParser so that
 # strict=True works well on Python 3.2.2.
 major, minor, release = sys.version_info[:3]
-CONSTRUCTOR_TAKES_STRICT = (
-    major > 3
-    or (major == 3 and minor > 2)
-    or (major == 3 and minor == 2 and release >= 3))
+CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3
+CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3
+CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4
+
 
 from bs4.element import (
     CData,
@@ -43,7 +53,42 @@ from bs4.builder import (
 HTMLPARSER = 'html.parser'
 
 class BeautifulSoupHTMLParser(HTMLParser):
-    def handle_starttag(self, name, attrs):
+
+    def __init__(self, *args, **kwargs):
+        HTMLParser.__init__(self, *args, **kwargs)
+
+        # Keep a list of empty-element tags that were encountered
+        # without an explicit closing tag. If we encounter a closing tag
+        # of this type, we'll associate it with one of those entries.
+        #
+        # This isn't a stack because we don't care about the
+        # order. It's a list of closing tags we've already handled and
+        # will ignore, assuming they ever show up.
+        self.already_closed_empty_element = []
+
+    def error(self, msg):
+        """In Python 3, HTMLParser subclasses must implement error(), although this
+        requirement doesn't appear to be documented.
+
+        In Python 2, HTMLParser implements error() as raising an exception.
+
+        In any event, this method is called only on very strange markup and our best strategy
+        is to pretend it didn't happen and keep going.
+        """
+        warnings.warn(msg)
+        
+    def handle_startendtag(self, name, attrs):
+        # This is only called when the markup looks like
+        # <tag/>.
+
+        # is_startend() tells handle_starttag not to close the tag
+        # just because its name matches a known empty-element tag. We
+        # know that this is an empty-element tag and we want to call
+        # handle_endtag ourselves.
+        tag = self.handle_starttag(name, attrs, handle_empty_element=False)
+        self.handle_endtag(name)
+        
+    def handle_starttag(self, name, attrs, handle_empty_element=True):
         # XXX namespace
         attr_dict = {}
         for key, value in attrs:
@@ -53,17 +98,46 @@ class BeautifulSoupHTMLParser(HTMLParser):
                 value = ''
             attr_dict[key] = value
             attrvalue = '""'
-        self.soup.handle_starttag(name, None, None, attr_dict)
+        #print "START", name
+        sourceline, sourcepos = self.getpos()
+        tag = self.soup.handle_starttag(
+            name, None, None, attr_dict, sourceline=sourceline,
+            sourcepos=sourcepos
+        )
+        if tag and tag.is_empty_element and handle_empty_element:
+            # Unlike other parsers, html.parser doesn't send separate end tag
+            # events for empty-element tags. (It's handled in
+            # handle_startendtag, but only if the original markup looked like
+            # <tag/>.)
+            #
+            # So we need to call handle_endtag() ourselves. Since we
+            # know the start event is identical to the end event, we
+            # don't want handle_endtag() to cross off any previous end
+            # events for tags of this name.
+            self.handle_endtag(name, check_already_closed=False)
 
-    def handle_endtag(self, name):
-        self.soup.handle_endtag(name)
+            # But we might encounter an explicit closing tag for this tag
+            # later on. If so, we want to ignore it.
+            self.already_closed_empty_element.append(name)
+            
+    def handle_endtag(self, name, check_already_closed=True):
+        #print "END", name
+        if check_already_closed and name in self.already_closed_empty_element:
+            # This is a redundant end tag for an empty-element tag.
+            # We've already called handle_endtag() for it, so just
+            # check it off the list.
+            # print "ALREADY CLOSED", name
+            self.already_closed_empty_element.remove(name)
+        else:
+            self.soup.handle_endtag(name)
 
     def handle_data(self, data):
         self.soup.handle_data(data)
 
     def handle_charref(self, name):
         # XXX workaround for a bug in HTMLParser. Remove this once
-        # it's fixed.
+        # it's fixed in all supported versions.
+        # http://bugs.python.org/issue13633
         if name.startswith('x'):
             real_name = int(name.lstrip('x'), 16)
         elif name.startswith('X'):
@@ -71,11 +145,26 @@ class BeautifulSoupHTMLParser(HTMLParser):
         else:
             real_name = int(name)
 
-        try:
-            data = unichr(real_name)
-        except (ValueError, OverflowError), e:
-            data = u"\N{REPLACEMENT CHARACTER}"
-
+        data = None
+        if real_name < 256:
+            # HTML numeric entities are supposed to reference Unicode
+            # code points, but sometimes they reference code points in
+            # some other encoding (ahem, Windows-1252). E.g. &#147;
+            # instead of &#201; for LEFT DOUBLE QUOTATION MARK. This
+            # code tries to detect this situation and compensate.
+            for encoding in (self.soup.original_encoding, 'windows-1252'):
+                if not encoding:
+                    continue
+                try:
+                    data = bytearray([real_name]).decode(encoding)
+                except UnicodeDecodeError as e:
+                    pass
+        if not data:
+            try:
+                data = chr(real_name)
+            except (ValueError, OverflowError) as e:
+                pass
+        data = data or "\N{REPLACEMENT CHARACTER}"
         self.handle_data(data)
 
     def handle_entityref(self, name):
@@ -83,7 +172,12 @@ class BeautifulSoupHTMLParser(HTMLParser):
         if character is not None:
             data = character
         else:
-            data = "&%s;" % name
+            # If this were XML, it would be ambiguous whether "&foo"
+            # was an character entity reference with a missing
+            # semicolon or the literal string "&foo". Since this is
+            # HTML, we have a complete list of all character entity references,
+            # and this one wasn't found, so assume it's the literal string "&foo".
+            data = "&%s" % name
         self.handle_data(data)
 
     def handle_comment(self, data):
@@ -113,14 +207,6 @@ class BeautifulSoupHTMLParser(HTMLParser):
 
     def handle_pi(self, data):
         self.soup.endData()
-        if data.endswith("?") and data.lower().startswith("xml"):
-            # "An XHTML processing instruction using the trailing '?'
-            # will cause the '?' to be included in data." - HTMLParser
-            # docs.
-            #
-            # Strip the question mark so we don't end up with two
-            # question marks.
-            data = data[:-1]
         self.soup.handle_data(data)
         self.soup.endData(ProcessingInstruction)
 
@@ -128,26 +214,38 @@ class BeautifulSoupHTMLParser(HTMLParser):
 class HTMLParserTreeBuilder(HTMLTreeBuilder):
 
     is_xml = False
-    features = [HTML, STRICT, HTMLPARSER]
+    picklable = True
+    NAME = HTMLPARSER
+    features = [NAME, HTML, STRICT]
 
-    def __init__(self, *args, **kwargs):
-        if CONSTRUCTOR_TAKES_STRICT:
-            kwargs['strict'] = False
-        self.parser_args = (args, kwargs)
+    # The html.parser knows which line number and position in the
+    # original file is the source of an element.
+    TRACKS_LINE_NUMBERS = True
+    
+    def __init__(self, parser_args=None, parser_kwargs=None, **kwargs):
+        super(HTMLParserTreeBuilder, self).__init__(**kwargs)
+        parser_args = parser_args or []
+        parser_kwargs = parser_kwargs or {}
+        if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED:
+            parser_kwargs['strict'] = False
+        if CONSTRUCTOR_TAKES_CONVERT_CHARREFS:
+            parser_kwargs['convert_charrefs'] = False
+        self.parser_args = (parser_args, parser_kwargs)
 
     def prepare_markup(self, markup, user_specified_encoding=None,
-                       document_declared_encoding=None):
+                       document_declared_encoding=None, exclude_encodings=None):
         """
         :return: A 4-tuple (markup, original encoding, encoding
         declared within markup, whether any characters had to be
         replaced with REPLACEMENT CHARACTER).
         """
-        if isinstance(markup, unicode):
+        if isinstance(markup, str):
             yield (markup, None, None, False)
             return
 
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
+        dammit = UnicodeDammit(markup, try_encodings, is_html=True,
+                               exclude_encodings=exclude_encodings)
         yield (dammit.markup, dammit.original_encoding,
                dammit.declared_html_encoding,
                dammit.contains_replacement_characters)
@@ -158,10 +256,12 @@ class HTMLParserTreeBuilder(HTMLTreeBuilder):
         parser.soup = self.soup
         try:
             parser.feed(markup)
-        except HTMLParseError, e:
+            parser.close()
+        except HTMLParseError as e:
             warnings.warn(RuntimeWarning(
                 "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
             raise e
+        parser.already_closed_empty_element = []
 
 # Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
 # 3.2.3 code. This ensures they don't treat markup like <p></p> as a

lib/bs4/builder/_lxml.py

@@ -1,13 +1,26 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'LXMLTreeBuilderForXML',
     'LXMLTreeBuilder',
     ]
 
+try:
+    from collections.abc import Callable # Python 3.6
+except ImportError as e:
+    from collections import Callable
+
 from io import BytesIO
-from StringIO import StringIO
-import collections
+from io import StringIO
 from lxml import etree
-from bs4.element import Comment, Doctype, NamespacedAttribute
+from bs4.element import (
+    Comment,
+    Doctype,
+    NamespacedAttribute,
+    ProcessingInstruction,
+    XMLProcessingInstruction,
+)
 from bs4.builder import (
     FAST,
     HTML,
@@ -20,19 +33,55 @@ from bs4.dammit import EncodingDetector
 
 LXML = 'lxml'
 
+def _invert(d):
+    "Invert a dictionary."
+    return dict((v,k) for k, v in list(d.items()))
+
 class LXMLTreeBuilderForXML(TreeBuilder):
     DEFAULT_PARSER_CLASS = etree.XMLParser
 
     is_xml = True
+    processing_instruction_class = XMLProcessingInstruction
+
+    NAME = "lxml-xml"
+    ALTERNATE_NAMES = ["xml"]
 
     # Well, it's permissive by XML parser standards.
-    features = [LXML, XML, FAST, PERMISSIVE]
+    features = [NAME, LXML, XML, FAST, PERMISSIVE]
 
     CHUNK_SIZE = 512
 
     # This namespace mapping is specified in the XML Namespace
     # standard.
-    DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"}
+    DEFAULT_NSMAPS = dict(xml='http://www.w3.org/XML/1998/namespace')
+
+    DEFAULT_NSMAPS_INVERTED = _invert(DEFAULT_NSMAPS)
+
+    # NOTE: If we parsed Element objects and looked at .sourceline,
+    # we'd be able to see the line numbers from the original document.
+    # But instead we build an XMLParser or HTMLParser object to serve
+    # as the target of parse messages, and those messages don't include
+    # line numbers.
+    
+    def initialize_soup(self, soup):
+        """Let the BeautifulSoup object know about the standard namespace
+        mapping.
+        """
+        super(LXMLTreeBuilderForXML, self).initialize_soup(soup)
+        self._register_namespaces(self.DEFAULT_NSMAPS)
+
+    def _register_namespaces(self, mapping):
+        """Let the BeautifulSoup object know about namespaces encountered
+        while parsing the document.
+
+        This might be useful later on when creating CSS selectors.
+        """
+        for key, value in list(mapping.items()):
+            if key and key not in self.soup._namespaces:
+                # Let the BeautifulSoup object know about a new namespace.
+                # If there are multiple namespaces defined with the same
+                # prefix, the first one in the document takes precedence.
+                self.soup._namespaces[key] = value
 
     def default_parser(self, encoding):
         # This can either return a parser object or a class, which
@@ -46,12 +95,12 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         # Use the default parser.
         parser = self.default_parser(encoding)
 
-        if isinstance(parser, collections.Callable):
+        if isinstance(parser, Callable):
             # Instantiate the parser with default arguments
             parser = parser(target=self, strip_cdata=False, encoding=encoding)
         return parser
 
-    def __init__(self, parser=None, empty_element_tags=None):
+    def __init__(self, parser=None, empty_element_tags=None, **kwargs):
         # TODO: Issue a warning if parser is present but not a
         # callable, since that means there's no way to create new
         # parsers for different encodings.
@@ -59,8 +108,9 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         if empty_element_tags is not None:
             self.empty_element_tags = set(empty_element_tags)
         self.soup = None
-        self.nsmaps = [self.DEFAULT_NSMAPS]
-
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
+        super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
+        
     def _getNsTag(self, tag):
         # Split the namespace URL out of a fully-qualified lxml tag
         # name. Copied from lxml's src/lxml/sax.py.
@@ -70,6 +120,7 @@ class LXMLTreeBuilderForXML(TreeBuilder):
             return (None, tag)
 
     def prepare_markup(self, markup, user_specified_encoding=None,
+                       exclude_encodings=None,
                        document_declared_encoding=None):
         """
         :yield: A series of 4-tuples.
@@ -78,31 +129,37 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
         Each 4-tuple represents a strategy for parsing the document.
         """
-        if isinstance(markup, unicode):
-            # We were given Unicode. Maybe lxml can parse Unicode on
-            # this system?
-            yield markup, None, document_declared_encoding, False
-
-        if isinstance(markup, unicode):
-            # No, apparently not. Convert the Unicode to UTF-8 and
-            # tell lxml to parse it as UTF-8.
-            yield (markup.encode("utf8"), "utf8",
-                   document_declared_encoding, False)
-
         # Instead of using UnicodeDammit to convert the bytestring to
         # Unicode using different encodings, use EncodingDetector to
         # iterate over the encodings, and tell lxml to try to parse
         # the document as each one in turn.
         is_html = not self.is_xml
+        if is_html:
+            self.processing_instruction_class = ProcessingInstruction
+        else:
+            self.processing_instruction_class = XMLProcessingInstruction
+
+        if isinstance(markup, str):
+            # We were given Unicode. Maybe lxml can parse Unicode on
+            # this system?
+            yield markup, None, document_declared_encoding, False
+
+        if isinstance(markup, str):
+            # No, apparently not. Convert the Unicode to UTF-8 and
+            # tell lxml to parse it as UTF-8.
+            yield (markup.encode("utf8"), "utf8",
+                   document_declared_encoding, False)
+
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        detector = EncodingDetector(markup, try_encodings, is_html)
+        detector = EncodingDetector(
+            markup, try_encodings, is_html, exclude_encodings)
         for encoding in detector.encodings:
             yield (detector.markup, encoding, document_declared_encoding, False)
 
     def feed(self, markup):
         if isinstance(markup, bytes):
             markup = BytesIO(markup)
-        elif isinstance(markup, unicode):
+        elif isinstance(markup, str):
             markup = StringIO(markup)
 
         # Call feed() at least once, even if the markup is empty,
@@ -117,30 +174,36 @@ class LXMLTreeBuilderForXML(TreeBuilder):
                 if len(data) != 0:
                     self.parser.feed(data)
             self.parser.close()
-        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
-            raise ParserRejectedMarkup(str(e))
+        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
+            raise ParserRejectedMarkup(e)
 
     def close(self):
-        self.nsmaps = [self.DEFAULT_NSMAPS]
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
 
     def start(self, name, attrs, nsmap={}):
         # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
         attrs = dict(attrs)
         nsprefix = None
         # Invert each namespace map as it comes in.
-        if len(self.nsmaps) > 1:
-            # There are no new namespaces for this tag, but
-            # non-default namespaces are in play, so we need a
-            # separate tag stack to know when they end.
-            self.nsmaps.append(None)
+        if len(nsmap) == 0 and len(self.nsmaps) > 1:
+                # There are no new namespaces for this tag, but
+                # non-default namespaces are in play, so we need a
+                # separate tag stack to know when they end.
+                self.nsmaps.append(None)
         elif len(nsmap) > 0:
             # A new namespace mapping has come into play.
-            inverted_nsmap = dict((value, key) for key, value in nsmap.items())
-            self.nsmaps.append(inverted_nsmap)
+
+            # First, Let the BeautifulSoup object know about it.
+            self._register_namespaces(nsmap)
+
+            # Then, add it to our running list of inverted namespace
+            # mappings.
+            self.nsmaps.append(_invert(nsmap))
+
             # Also treat the namespace mapping as a set of attributes on the
             # tag, so we can recreate it later.
             attrs = attrs.copy()
-            for prefix, namespace in nsmap.items():
+            for prefix, namespace in list(nsmap.items()):
                 attribute = NamespacedAttribute(
                     "xmlns", prefix, "http://www.w3.org/2000/xmlns/")
                 attrs[attribute] = namespace
@@ -149,7 +212,7 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         # from lxml with namespaces attached to their names, and
         # turn then into NamespacedAttribute objects.
         new_attrs = {}
-        for attr, value in attrs.items():
+        for attr, value in list(attrs.items()):
             namespace, attr = self._getNsTag(attr)
             if namespace is None:
                 new_attrs[attr] = value
@@ -189,7 +252,9 @@ class LXMLTreeBuilderForXML(TreeBuilder):
             self.nsmaps.pop()
 
     def pi(self, target, data):
-        pass
+        self.soup.endData()
+        self.soup.handle_data(target + ' ' + data)
+        self.soup.endData(self.processing_instruction_class)
 
     def data(self, content):
         self.soup.handle_data(content)
@@ -207,13 +272,17 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""
-        return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
+        return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
 
 
 class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
 
-    features = [LXML, HTML, FAST, PERMISSIVE]
+    NAME = LXML
+    ALTERNATE_NAMES = ["lxml-html"]
+
+    features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE]
     is_xml = False
+    processing_instruction_class = ProcessingInstruction
 
     def default_parser(self, encoding):
         return etree.HTMLParser
@@ -224,10 +293,10 @@ class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
             self.parser = self.parser_for(encoding)
             self.parser.feed(markup)
             self.parser.close()
-        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
-            raise ParserRejectedMarkup(str(e))
+        except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
+            raise ParserRejectedMarkup(e)
 
 
     def test_fragment_to_document(self, fragment):
         """See `TreeBuilder`."""
-        return u'<html><body>%s</body></html>' % fragment
+        return '<html><body>%s</body></html>' % fragment

lib/bs4/check_block.py

@@ -0,0 +1,4 @@
+import requests
+data = requests.get("https://www.crummy.com/").content
+from bs4 import _s
+data = [x for x in _s(data).block_text()]

lib/bs4/dammit.py

@@ -3,12 +3,15 @@
 
 This library converts a bytestream to Unicode through any means
 necessary. It is heavily based on code from Mark Pilgrim's Universal
-Feed Parser. It works best on XML and XML, but it does not rewrite the
+Feed Parser. It works best on XML and HTML, but it does not rewrite the
 XML or HTML to reflect a new encoding; that's the tree builder's job.
 """
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
 
 import codecs
-from htmlentitydefs import codepoint2name
+from future.moves.html.entities import codepoint2name
+from future.builtins import chr
 import re
 import logging
 import string
@@ -20,6 +23,8 @@ try:
     #  PyPI package: cchardet
     import cchardet
     def chardet_dammit(s):
+        if isinstance(s, str):
+            return None
         return cchardet.detect(s)['encoding']
 except ImportError:
     try:
@@ -28,6 +33,8 @@ except ImportError:
         #  PyPI package: chardet
         import chardet
         def chardet_dammit(s):
+            if isinstance(s, str):
+                return None
             return chardet.detect(s)['encoding']
         #import chardet.constants
         #chardet.constants._debug = 1
@@ -42,10 +49,19 @@ try:
 except ImportError:
     pass
 
-xml_encoding_re = re.compile(
-    '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
-html_meta_re = re.compile(
-    '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
+# Build bytestring and Unicode versions of regular expressions for finding
+# a declared encoding inside an XML or HTML document.
+xml_encoding = '^\s*<\\?.*encoding=[\'"](.*?)[\'"].*\\?>'
+html_meta = '<\\s*meta[^>]+charset\\s*=\\s*["\']?([^>]*?)[ /;\'">]'
+encoding_res = dict()
+encoding_res[bytes] = {
+    'html' : re.compile(html_meta.encode("ascii"), re.I),
+    'xml' : re.compile(xml_encoding.encode("ascii"), re.I),
+}
+encoding_res[str] = {
+    'html' : re.compile(html_meta, re.I),
+    'xml' : re.compile(xml_encoding, re.I)
+}
 
 class EntitySubstitution(object):
 
@@ -55,15 +71,24 @@ class EntitySubstitution(object):
         lookup = {}
         reverse_lookup = {}
         characters_for_re = []
-        for codepoint, name in list(codepoint2name.items()):
-            character = unichr(codepoint)
-            if codepoint != 34:
+
+        # &apos is an XHTML entity and an HTML 5, but not an HTML 4
+        # entity. We don't want to use it, but we want to recognize it on the way in.
+        #
+        # TODO: Ideally we would be able to recognize all HTML 5 named
+        # entities, but that's a little tricky.
+        extra = [(39, 'apos')]
+        for codepoint, name in list(codepoint2name.items()) + extra:
+            character = chr(codepoint)
+            if codepoint not in (34, 39):
                 # There's no point in turning the quotation mark into
-                # &quot;, unless it happens within an attribute value, which
-                # is handled elsewhere.
+                # &quot; or the single quote into &apos;, unless it
+                # happens within an attribute value, which is handled
+                # elsewhere.
                 characters_for_re.append(character)
                 lookup[character] = name
-            # But we do want to turn &quot; into the quotation mark.
+            # But we do want to recognize those entities on the way in and
+            # convert them to Unicode characters.
             reverse_lookup[name] = character
         re_definition = "[%s]" % "".join(characters_for_re)
         return lookup, reverse_lookup, re.compile(re_definition)
@@ -79,7 +104,7 @@ class EntitySubstitution(object):
         }
 
     BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
-                                           "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
+                                           "&(?!#\\d+;|#x[0-9a-fA-F]+;|\\w+;)"
                                            ")")
 
     AMPERSAND_OR_BRACKET = re.compile("([<>&])")
@@ -212,8 +237,11 @@ class EncodingDetector:
 
     5. Windows-1252.
     """
-    def __init__(self, markup, override_encodings=None, is_html=False):
+    def __init__(self, markup, override_encodings=None, is_html=False,
+                 exclude_encodings=None):
         self.override_encodings = override_encodings or []
+        exclude_encodings = exclude_encodings or []
+        self.exclude_encodings = set([x.lower() for x in exclude_encodings])
         self.chardet_encoding = None
         self.is_html = is_html
         self.declared_encoding = None
@@ -224,6 +252,8 @@ class EncodingDetector:
     def _usable(self, encoding, tried):
         if encoding is not None:
             encoding = encoding.lower()
+            if encoding in self.exclude_encodings:
+                return False
             if encoding not in tried:
                 tried.add(encoding)
                 return True
@@ -266,6 +296,9 @@ class EncodingDetector:
     def strip_byte_order_mark(cls, data):
         """If a byte-order mark is present, strip it and return the encoding it implies."""
         encoding = None
+        if isinstance(data, str):
+            # Unicode data cannot have a byte-order mark.
+            return data, encoding
         if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
                and (data[2:4] != '\x00\x00'):
             encoding = 'utf-16be'
@@ -300,14 +333,22 @@ class EncodingDetector:
             xml_endpos = 1024
             html_endpos = max(2048, int(len(markup) * 0.05))
 
+        if isinstance(markup, bytes):
+            res = encoding_res[bytes]
+        else:
+            res = encoding_res[str]
+
+        xml_re = res['xml']
+        html_re = res['html']
         declared_encoding = None
-        declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos)
+        declared_encoding_match = xml_re.search(markup, endpos=xml_endpos)
         if not declared_encoding_match and is_html:
-            declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos)
+            declared_encoding_match = html_re.search(markup, endpos=html_endpos)
         if declared_encoding_match is not None:
-            declared_encoding = declared_encoding_match.groups()[0].decode(
-                'ascii')
+            declared_encoding = declared_encoding_match.groups()[0]
         if declared_encoding:
+            if isinstance(declared_encoding, bytes):
+                declared_encoding = declared_encoding.decode('ascii', 'replace')
             return declared_encoding.lower()
         return None
 
@@ -331,18 +372,19 @@ class UnicodeDammit:
         ]
 
     def __init__(self, markup, override_encodings=[],
-                 smart_quotes_to=None, is_html=False):
+                 smart_quotes_to=None, is_html=False, exclude_encodings=[]):
         self.smart_quotes_to = smart_quotes_to
         self.tried_encodings = []
         self.contains_replacement_characters = False
         self.is_html = is_html
-
-        self.detector = EncodingDetector(markup, override_encodings, is_html)
+        self.log = logging.getLogger(__name__)
+        self.detector = EncodingDetector(
+            markup, override_encodings, is_html, exclude_encodings)
 
         # Short-circuit if the data is in Unicode to begin with.
-        if isinstance(markup, unicode) or markup == '':
+        if isinstance(markup, str) or markup == '':
             self.markup = markup
-            self.unicode_markup = unicode(markup)
+            self.unicode_markup = str(markup)
             self.original_encoding = None
             return
 
@@ -365,9 +407,10 @@ class UnicodeDammit:
                 if encoding != "ascii":
                     u = self._convert_from(encoding, "replace")
                 if u is not None:
-                    logging.warning(
+                    self.log.warning(
                             "Some characters could not be decoded, and were "
-                            "replaced with REPLACEMENT CHARACTER.")
+                            "replaced with REPLACEMENT CHARACTER."
+                    )
                     self.contains_replacement_characters = True
                     break
 
@@ -425,7 +468,7 @@ class UnicodeDammit:
     def _to_unicode(self, data, encoding, errors="strict"):
         '''Given a string and its encoding, decodes the string into Unicode.
         %encoding is a string recognized by encodings.aliases'''
-        return unicode(data, encoding, errors)
+        return str(data, encoding, errors)
 
     @property
     def declared_html_encoding(self):
@@ -723,7 +766,7 @@ class UnicodeDammit:
         0xde : b'\xc3\x9e',     # Þ
         0xdf : b'\xc3\x9f',     # ß
         0xe0 : b'\xc3\xa0',     # à
-        0xe1 : b'\xa1',     # á
+        0xe1 : b'\xa1',         # á
         0xe2 : b'\xc3\xa2',     # â
         0xe3 : b'\xc3\xa3',     # ã
         0xe4 : b'\xc3\xa4',     # ä

lib/bs4/diagnose.py

@@ -1,7 +1,11 @@
 """Diagnostic functions, mainly for use when doing tech support."""
+
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 import cProfile
-from StringIO import StringIO
-from HTMLParser import HTMLParser
+from io import StringIO
+from html.parser import HTMLParser
 import bs4
 from bs4 import BeautifulSoup, __version__
 from bs4.builder import builder_registry
@@ -17,8 +21,8 @@ import cProfile
 
 def diagnose(data):
     """Diagnostic suite for isolating common problems."""
-    print "Diagnostic running on Beautiful Soup %s" % __version__
-    print "Python version %s" % sys.version
+    print("Diagnostic running on Beautiful Soup %s" % __version__)
+    print("Python version %s" % sys.version)
 
     basic_parsers = ["html.parser", "html5lib", "lxml"]
     for name in basic_parsers:
@@ -27,44 +31,60 @@ def diagnose(data):
                 break
         else:
             basic_parsers.remove(name)
-            print (
+            print((
                 "I noticed that %s is not installed. Installing it may help." %
-                name)
+                name))
 
     if 'lxml' in basic_parsers:
-        basic_parsers.append(["lxml", "xml"])
-        from lxml import etree
-        print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))
+        basic_parsers.append("lxml-xml")
+        try:
+            from lxml import etree
+            print("Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)))
+        except ImportError as e:
+            print (
+                "lxml is not installed or couldn't be imported.")
+
 
     if 'html5lib' in basic_parsers:
-        import html5lib
-        print "Found html5lib version %s" % html5lib.__version__
+        try:
+            import html5lib
+            print("Found html5lib version %s" % html5lib.__version__)
+        except ImportError as e:
+            print (
+                "html5lib is not installed or couldn't be imported.")
 
     if hasattr(data, 'read'):
         data = data.read()
-    elif os.path.exists(data):
-        print '"%s" looks like a filename. Reading data from the file.' % data
-        data = open(data).read()
     elif data.startswith("http:") or data.startswith("https:"):
-        print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data
-        print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup."
+        print('"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data)
+        print("You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup.")
         return
-    print
+    else:
+        try:
+            if os.path.exists(data):
+                print('"%s" looks like a filename. Reading data from the file.' % data)
+                with open(data) as fp:
+                    data = fp.read()
+        except ValueError:
+            # This can happen on some platforms when the 'filename' is
+            # too long. Assume it's data and not a filename.
+            pass
+        print()
 
     for parser in basic_parsers:
-        print "Trying to parse your markup with %s" % parser
+        print("Trying to parse your markup with %s" % parser)
         success = False
         try:
-            soup = BeautifulSoup(data, parser)
+            soup = BeautifulSoup(data, features=parser)
             success = True
-        except Exception, e:
-            print "%s could not parse the markup." % parser
+        except Exception as e:
+            print("%s could not parse the markup." % parser)
             traceback.print_exc()
         if success:
-            print "Here's what %s did with the markup:" % parser
-            print soup.prettify()
+            print("Here's what %s did with the markup:" % parser)
+            print(soup.prettify())
 
-        print "-" * 80
+        print("-" * 80)
 
 def lxml_trace(data, html=True, **kwargs):
     """Print out the lxml events that occur during parsing.
@@ -74,7 +94,7 @@ def lxml_trace(data, html=True, **kwargs):
     """
     from lxml import etree
     for event, element in etree.iterparse(StringIO(data), html=html, **kwargs):
-        print("%s, %4s, %s" % (event, element.tag, element.text))
+        print(("%s, %4s, %s" % (event, element.tag, element.text)))
 
 class AnnouncingParser(HTMLParser):
     """Announces HTMLParser parse events, without doing anything else."""
@@ -135,7 +155,7 @@ def rword(length=5):
 def rsentence(length=4):
     "Generate a random sentence-like string."
     return " ".join(rword(random.randint(4,9)) for i in range(length))
-
+        
 def rdoc(num_elements=1000):
     """Randomly generate an invalid HTML document."""
     tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table']
@@ -156,10 +176,10 @@ def rdoc(num_elements=1000):
 
 def benchmark_parsers(num_elements=100000):
     """Very basic head-to-head performance benchmark."""
-    print "Comparative parser benchmark on Beautiful Soup %s" % __version__
+    print("Comparative parser benchmark on Beautiful Soup %s" % __version__)
     data = rdoc(num_elements)
-    print "Generated a large invalid HTML document (%d bytes)." % len(data)
-
+    print("Generated a large invalid HTML document (%d bytes)." % len(data))
+    
     for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
         success = False
         try:
@@ -167,24 +187,24 @@ def benchmark_parsers(num_elements=100000):
             soup = BeautifulSoup(data, parser)
             b = time.time()
             success = True
-        except Exception, e:
-            print "%s could not parse the markup." % parser
+        except Exception as e:
+            print("%s could not parse the markup." % parser)
             traceback.print_exc()
         if success:
-            print "BS4+%s parsed the markup in %.2fs." % (parser, b-a)
+            print("BS4+%s parsed the markup in %.2fs." % (parser, b-a))
 
     from lxml import etree
     a = time.time()
     etree.HTML(data)
     b = time.time()
-    print "Raw lxml parsed the markup in %.2fs." % (b-a)
+    print("Raw lxml parsed the markup in %.2fs." % (b-a))
 
     import html5lib
     parser = html5lib.HTMLParser()
     a = time.time()
     parser.parse(data)
     b = time.time()
-    print "Raw html5lib parsed the markup in %.2fs." % (b-a)
+    print("Raw html5lib parsed the markup in %.2fs." % (b-a))
 
 def profile(num_elements=100000, parser="lxml"):