v2.1.0-beta

* Remove individual media type toggles

.gitignore

@@ -15,7 +15,9 @@
 release.lock
 version.lock
 logs/*
+backups/*
 cache/*
+newsletters/*
 *.mmdb
 
 # HTTPS Cert/Key #

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## v2.1.0-beta (2018-04-07)
+
+* Newsletters:
+  * New: A completely new scheduled newsletter system.
+    * Beautiful HTML formatted newsletter for recently added movies, TV shows, or music.
+    * Send newsletters on a daily, weekly, or monthly schedule to your users.
+    * Customize the number of days of recently added content and the libraries to include on the newsletter.
+    * Add a custom message to be included on the newsletter.
+    * Option to either send an HTML formatted email, or a link to a self-hosted newsletter on your own domain to any notification agent.
+* Notifications:
+  * New: Ability to use self-hosted images on your own domain instead of using Imgur.
+
+
+## v2.0.28 (2018-04-02)
+
+* Monitoring:
+  * Fix: Homepage activity header text.
+
+
+## v2.0.27 (2018-04-02)
+
+* Monitoring:
+  * Change: Move activity refresh interval setting to the settings page.
+
+
 ## v2.0.26-beta (2018-03-30)
 
 * Monitoring:

data/interfaces/default/configuration_table.html

@@ -49,6 +49,10 @@ DOCUMENTATION :: END
             <td>Cache Directory:</td>
             <td>${plexpy.CONFIG.CACHE_DIR}</td>
         </tr>
+        <tr>
+            <td>Newsletter Directory:</td>
+            <td>${plexpy.CONFIG.NEWSLETTER_DIR}</td>
+        </tr>
         <tr>
             <td>GeoLite2 Database:</td>
             % if plexpy.CONFIG.GEOIP_DB:

data/interfaces/default/css/tautulli.css

@@ -125,8 +125,10 @@ div.form-control .selectize-input {
     padding-bottom: 2px !important;
     transition: background-color .3s;
 }
-.react-selectize.root-node .simple-value span {
+.react-selectize.root-node .simple-value span,
+.selectize-control.multi .selectize-input > div {
     padding-bottom: 2px !important;
+    padding-left: 5px !important;
 }
 .react-selectize.root-node .react-selectize-control .react-selectize-search-field-and-selected-values .value-wrapper:not(:first-child):before {
     content: "or";
@@ -294,6 +296,10 @@ object {
     font-weight: bold;
     text-transform: uppercase;
 }
+.padded-header h3 small {
+    font-size: 13px;
+    text-transform: none;
+}
 .btn {
     outline:0px !important;
 }
@@ -460,6 +466,18 @@ fieldset[disabled] .btn-bright.active {
 .btn-group select {
     margin-top: 0;
 }
+.input-group-addon-form {
+    display: inline-block;
+    line-height: 1.42857143;
+    color: #e5e5e5;
+    background-color: #3B3B3B;
+    border: 1px solid transparent;
+    border-top-right-radius: 3px !important;
+    border-bottom-right-radius: 3px !important;
+    height: 32px;
+    width: 100%;
+    margin-top: 5px;
+}
 #user-selection label {
     margin-bottom: 0;
 }
@@ -738,7 +756,10 @@ a .users-poster-face:hover {
     transition: all .2s ease-in-out;
     overflow: hidden;
 }
-.dashboard-activity-background-overlay {
+.dashboard-activity-background {
+    background-color: #282828;
+    background-position: center;
+    background-size: cover;
     display: -webkit-flex;
     display: flex;
     -webkit-flex-wrap: nowrap;
@@ -747,30 +768,13 @@ a .users-poster-face:hover {
     width: 100%;
     padding: 5px;
     overflow: hidden;
-    -webkit-box-shadow: 0 0 4px rgba(0,0,0,.3), inset 0 0 0 1px rgba(255,255,255,.1);
-    -moz-box-shadow: 0 0 4px rgba(0,0,0,.3),inset 0 0 0 1px rgba(255,255,255,.1);
-    box-shadow: 0 0 4px rgba(0,0,0,.3), inset 0 0 0 1px rgba(255,255,255,.1);
-}
-.dashboard-activity-background {
-    background-color: #282828;
-    background-position: center;
-    background-size: cover;
-    height: 235px;
-    width: 100%;
-    position: absolute;
-    top: 0;
-    right: 0;
-    bottom: 0;
-    left: 0;
-    opacity: 0.40;
-    -webkit-filter: blur(3px);
-    -moz-filter: blur(3px);
-    filter: blur(3px);
     -webkit-transition: background 1s linear;
     transition: background 1s linear;
     -webkit-backface-visibility: hidden;
     backface-visibility: hidden;
-    z-index: -1;
+    -webkit-box-shadow: 0 0 4px rgba(0,0,0,.3), inset 0 0 0 1px rgba(255,255,255,.1);
+    -moz-box-shadow: 0 0 4px rgba(0,0,0,.3),inset 0 0 0 1px rgba(255,255,255,.1);
+    box-shadow: 0 0 4px rgba(0,0,0,.3), inset 0 0 0 1px rgba(255,255,255,.1);
 }
 .dashboard-activity-poster-container {
     background-color: #282828;
@@ -801,14 +805,14 @@ a .users-poster-face:hover {
     background-size: cover;
     height: 225px;
     width: 150px;
-    -webkit-transition: background .2s ease-in-out;
-    transition: background .2s ease-in-out;
-    -webkit-backface-visibility: hidden;
-    backface-visibility: hidden;
     opacity: 0.60;
     -webkit-filter: blur(3px);
     -moz-filter: blur(3px);
     filter: blur(3px);
+    -webkit-transition: background .2s ease-in-out;
+    transition: background .2s ease-in-out;
+    -webkit-backface-visibility: hidden;
+    backface-visibility: hidden;
     z-index: 2;
 }
 .dashboard-activity-cover {
@@ -1155,7 +1159,10 @@ a .dashboard-activity-metadata-user-thumb:hover {
     transition: all .2s ease-in-out;
     overflow: hidden;
 }
-.dashboard-stats-background-overlay {
+.dashboard-stats-background {
+    background-color: #282828;
+    background-position: center;
+    background-size: cover;
     display: -webkit-flex;
     display: flex;
     -webkit-flex-wrap: nowrap;
@@ -1164,30 +1171,13 @@ a .dashboard-activity-metadata-user-thumb:hover {
     width: 100%;
     padding: 5px;
     overflow: hidden;
-    -webkit-box-shadow: 0 0 4px rgba(0,0,0,.3), inset 0 0 0 1px rgba(255,255,255,.1);
-    -moz-box-shadow: 0 0 4px rgba(0,0,0,.3),inset 0 0 0 1px rgba(255,255,255,.1);
-    box-shadow: 0 0 4px rgba(0,0,0,.3), inset 0 0 0 1px rgba(255,255,255,.1);
-}
-.dashboard-stats-background {
-    background-color: #282828;
-    background-position: center;
-    background-size: cover;
-    height: 160px;
-    width: 100%;
-    position: absolute;
-    top: 0;
-    right: 0;
-    bottom: 0;
-    left: 0;
-    opacity: 0.40;
-    -webkit-filter: blur(3px);
-    -moz-filter: blur(3px);
-    filter: blur(3px);
     -webkit-transition: background .2s ease-in-out;
     transition: background .2s ease-in-out;
     -webkit-backface-visibility: hidden;
     backface-visibility: hidden;
-    z-index: -1;
+    -webkit-box-shadow: 0 0 4px rgba(0,0,0,.3), inset 0 0 0 1px rgba(255,255,255,.1);
+    -moz-box-shadow: 0 0 4px rgba(0,0,0,.3),inset 0 0 0 1px rgba(255,255,255,.1);
+    box-shadow: 0 0 4px rgba(0,0,0,.3), inset 0 0 0 1px rgba(255,255,255,.1);
 }
 .dashboard-stats-background.flat {
     opacity: 1;
@@ -1207,17 +1197,6 @@ a .dashboard-activity-metadata-user-thumb:hover {
     z-index: 1;
 }
 .dashboard-stats-poster {
-    background-position: center;
-    background-size: cover;
-    height: 150px;
-    width: 100px;
-    -webkit-transition: background .2s ease-in-out;
-    transition: background .2s ease-in-out;
-    -webkit-backface-visibility: hidden;
-    backface-visibility: hidden;
-    z-index: 2;
-}
-.dashboard-stats-poster-blur {
     background-color: #282828;
     background-position: center;
     background-size: cover;
@@ -1227,10 +1206,6 @@ a .dashboard-activity-metadata-user-thumb:hover {
     transition: background .2s ease-in-out;
     -webkit-backface-visibility: hidden;
     backface-visibility: hidden;
-    opacity: 0.60;
-    -webkit-filter: blur(3px);
-    -moz-filter: blur(3px);
-    filter: blur(3px);
     z-index: 2;
 }
 .dashboard-stats-cover {
@@ -2153,6 +2128,12 @@ a:hover .item-children-poster {
     top: 5px;
     left: 12px;
 }
+.settings-warning {
+    color: #eb8600;
+}
+span.settings-warning {
+    padding-left: 10px;
+}
 #menu_link_show_advanced_settings.active {
   color: #fff;
   background-color: #cc7b19;
@@ -2373,18 +2354,6 @@ a .library-user-instance-box:hover {
     margin-top: 9px;
     width: 175px;
 }
-.home-padded-header .info-bar {
-    float: left;
-    margin-left: 15px;
-    line-height: 35px;
-}
-.home-padded-header .info-bar small {
-    font-size: 13px;
-    font-weight: normal;
-    text-transform: none;
-    line-height: 1;
-    color: #777;
-}
 .home-padded-header .button-bar {
     float: left;
 }
@@ -2978,6 +2947,9 @@ a .home-platforms-list-cover-face:hover
 .stacked-configs > li.new-notification-agent,
 .stacked-configs > li.notification-agent,
 .stacked-configs > li.add-notification-agent,
+.stacked-configs > li.new-newsletter-agent,
+.stacked-configs > li.newsletter-agent,
+.stacked-configs > li.add-newsletter-agent,
 .stacked-configs > li.mobile-device,
 .stacked-configs > li.add-mobile-device {
     cursor: pointer;
@@ -3662,38 +3634,71 @@ a:hover .overlay-refresh-image:hover {
 }
 #plexpy-notifiers-table .friendly_name,
 #notifier-config-modal span.notifier_id,
+#plexpy-newsletters-table .friendly_name,
+#newsletter-config-modal span.newsletter_id,
 #plexpy-mobile-devices-table .friendly_name,
 #mobile-device-config-modal span.notifier_id {
     color: #777;
 }
-#notifier-config-modal .nav-tabs {
+#notifier-config-modal .nav-tabs,
+#newsletter-config-modal .nav-tabs {
     margin-bottom: 10px;
     padding-left: 15px;
     border-bottom: 1px solid #444;
 }
-#notifier-config-modal .nav-tabs > li {
+#notifier-config-modal .nav-tabs > li,
+#newsletter-config-modal .nav-tabs > li {
     margin: 0 0 -1px 0;
 }
-#notifier-config-modal .nav-tabs > li > a {
+#notifier-config-modal .nav-tabs > li > a,
+#newsletter-config-modal .nav-tabs > li > a {
     padding: 5px 10px;
     color: #737373;
 }
-#notifier-config-modal .nav-tabs > li > a:hover {
+#notifier-config-modal .nav-tabs > li > a:hover,
+#newsletter-config-modal .nav-tabs > li > a:hover {
     border-color: #444;
     background: #222;
 }
 #notifier-config-modal .nav-tabs > li.active > a,
 #notifier-config-modal .nav-tabs > li.active > a:hover,
-#notifier-config-modal .nav-tabs > li.active > a:focus {
+#notifier-config-modal .nav-tabs > li.active > a:focus,
+#newsletter-config-modal .nav-tabs > li.active > a,
+#newsletter-config-modal .nav-tabs > li.active > a:hover,
+#newsletter-config-modal .nav-tabs > li.active > a:focus {
     color: #fff;
     background: #222;
 }
 #notifier-config-modal .nav-tabs > li.active > a,
 #notifier-config-modal .nav-tabs > li.active > a:hover,
-#notifier-config-modal .nav-tabs > li.active > a:focus {
+#notifier-config-modal .nav-tabs > li.active > a:focus,
+#newsletter-config-modal .nav-tabs > li.active > a,
+#newsletter-config-modal .nav-tabs > li.active > a:hover,
+#newsletter-config-modal .nav-tabs > li.active > a:focus {
     border: 1px solid #444;
     border-bottom-color: transparent;
 }
+#newsletter-config-modal #custom_cron {
+    display: inline-block;
+    width: initial;
+    height: 32px;
+    margin-right: 5px;
+    margin-top: 4px;
+}
+#newsletter-config-modal #cron-widget {
+    display: inline-block;
+    margin-top: 1px;
+}
+#newsletter-config-modal #cron-widget select.cron-select {
+    width: initial;
+    display: inline;
+    height: 32px;
+    margin-top: 4px;
+}
+#newsletter-config-modal #cron-widget select.cron-select[name=cron-period] option[value=minute],
+#newsletter-config-modal #cron-widget select.cron-select[name=cron-period] option[value=hour] {
+    display: none !important;
+}
 .git-group input.form-control {
     width: 50%;
 }
@@ -3846,6 +3851,90 @@ a:hover .overlay-refresh-image:hover {
     background-color: #107c10;
     background-image: url(../images/platforms/xbox.svg);
 }
+.platform-android-rgba {
+    background-color: rgba(164, 202, 57, 0.40);
+}
+.platform-atv-rgba {
+    background-color: rgba(133, 132, 135, 0.40);
+}
+.platform-chrome-rgba {
+    background-color: rgba(237, 94, 80, 0.40);
+}
+.platform-chromecast-rgba {
+    background-color: rgba(16, 164, 232, 0.40);
+}
+.platform-default-rgba {
+    background-color: rgba(229, 160, 13, 0.40);
+}
+.platform-dlna-rgba {
+    background-color: rgba(12, 177, 75, 0.40);
+}
+.platform-firefox-rgba {
+    background-color: rgba(230, 120, 23, 0.40);
+}
+.platform-gtv-rgba {
+    background-color: rgba(0, 139, 207, 0.40);
+}
+.platform-ie-rgba {
+    background-color: rgba(0, 89, 158, 0.40);
+}
+.platform-ios-rgba {
+    background-color: rgba(133, 132, 135, 0.40);
+}
+.platform-kodi-rgba {
+    background-color: rgba(49, 175, 225, 0.40);
+}
+.platform-linux-rgba {
+    background-color: rgba(23, 147, 208, 0.40);
+}
+.platform-macos-rgba {
+    background-color: rgba(133, 132, 135, 0.40);
+}
+.platform-msedge-rgba {
+    background-color: rgba(0, 120, 215, 0.40);
+}
+.platform-opera-rgba {
+    background-color: rgba(255, 27, 45, 0.40);
+}
+.platform-playstation-rgba {
+    background-color: rgba(3, 77, 162, 0.40);
+}
+.platform-plex-rgba {
+    background-color: rgba(229, 160, 13, 0.40);
+}
+.platform-plexamp-rgba {
+    background-color: rgba(229, 160, 13, 0.40);
+}
+.platform-roku-rgba {
+    background-color: rgba(109, 60, 151, 0.40);
+}
+.platform-safari-rgba {
+    background-color: rgba(0, 169, 236, 0.40);
+}
+.platform-samsung-rgba {
+    background-color: rgba(3, 78, 162, 0.40);
+}
+.platform-synclounge-rgba {
+    background-color: rgba(21, 25, 36, 0.40);
+}
+.platform-tivo-rgba {
+    background-color: rgba(0, 167, 225, 0.40);
+}
+.platform-wiiu-rgba {
+    background-color: rgba(3, 169, 244, 0.40);
+}
+.platform-windows-rgba {
+    background-color: rgba(47, 192, 245, 0.40);
+}
+.platform-wp-rgba {
+    background-color: rgba(104, 33, 122, 0.40);
+}
+.platform-xbmc-rgba {
+    background-color: rgba(59, 72, 114, 0.40);
+}
+.platform-xbox-rgba {
+    background-color: rgba(16, 124, 16, 0.40);
+}
 .library-movie {
     background-image: url(../images/libraries/movie.svg);
 }
@@ -3957,3 +4046,37 @@ a:hover .overlay-refresh-image:hover {
     -webkit-appearance: none;
     margin: 0;
 }
+.newsletter-loader-container {
+    font-family: 'Open Sans', Arial, sans-serif;
+    position: absolute;
+    top: 0;
+    right: 0;
+    bottom: 0;
+    left: 0;
+}
+.newsletter-loader-message {
+    color: #282A2D;
+    text-align: center;
+    position: absolute;
+    left: 50%;
+    top: 25%;
+    transform: translate(-50%, -50%);
+}
+.newsletter-loader {
+    border: 5px solid #ccc;
+    -webkit-animation: spin 1s linear infinite;
+    animation: spin 1s linear infinite;
+    border-top: 5px solid #282A2D;
+    border-radius: 50%;
+    width: 50px;
+    height: 50px;
+    position: relative;
+    left: calc(50% - 25px);
+}
+@keyframes spin {
+    0% { transform: rotate(0deg); }
+    100% { transform: rotate(360deg); }
+}
+a[data-tab-destination] {
+    cursor: pointer;
+}

data/interfaces/default/current_activity_instance.html

@@ -79,20 +79,19 @@ DOCUMENTATION :: END
 <div class="dashboard-activity-instance" id="activity-instance-${sk}" data-key="${sk}" data-id="${data['session_id']}"
      data-rating_key="${data['rating_key']}" data-parent_rating_key="${data['parent_rating_key']}" data-grandparent_rating_key="${data['grandparent_rating_key']}">
     <div class="dashboard-activity-container">
-        <div class="dashboard-activity-background-overlay">
-            % if data['channel_stream'] == 0:
-            <div id="background-${sk}" class="dashboard-activity-background" style="background-image: url(pms_image_proxy?img=${data['art']}&width=500&height=280&fallback=art&refresh=true);"></div>
-            % else:
-                % if (data['art'] and data['art'].startswith('http')) or (data['thumb'] and data['thumb'].startswith('http')):
-                <div id="background-${sk}" class="dashboard-activity-background" style="background-image: url(${data['art']});"></div>
-                % else:
-                <!--Hacky solution to escape the image url until I come up with something better-->
-                <div id="background-${sk}" class="dashboard-activity-background" style="background-image: url(pms_image_proxy?img=${quote(data['art'] or data['thumb'])}&width=500&height=280&fallback=art&refresh=true&clip=true);"></div>
-                % endif
-            % endif
+        <%
+            if data['channel_stream'] == 0:
+                background_url = 'pms_image_proxy?img=' + data['art'] + '&width=500&height=280&opacity=40&background=282828&blur=3&fallback=art&refresh=true'
+            else:
+                if (data['art'] and data['art'].startswith('http')) or (data['thumb'] and data['thumb'].startswith('http')):
+                    background_url = data['art']
+                else:
+                    background_url = 'pms_image_proxy?img=' + quote(data['art'] or data['thumb']) + '&width=500&height=280&fallback=art&refresh=true&clip=true'
+        %>
+        <div id="background-${sk}" class="dashboard-activity-background" style="background-image: url(${background_url});">
             <div class="dashboard-activity-poster-container hidden-xs">
                 % if data['media_type'] == 'track':
-                <div id="poster-${sk}-bg" class="dashboard-activity-poster-blur" style="background-image: url(pms_image_proxy?img=${data['parent_thumb']}&width=300&height=300&fallback=cover&refresh=true);"></div>
+                <div id="poster-${sk}-bg" class="dashboard-activity-poster" style="background-image: url(pms_image_proxy?img=${data['parent_thumb']}&width=300&height=300&opacity=60&background=282828&blur=3&fallback=cover&refresh=true);"></div>
                 % endif
                 % if data['channel_stream'] == 0:
                     % if data['media_type'] == 'movie':
@@ -121,7 +120,7 @@ DOCUMENTATION :: END
                     <div id="poster-${sk}" class="dashboard-activity-poster-blur" style="background-image: url(${data['channel_icon']});"></div>
                     <div id="poster-${sk}" class="dashboard-activity-cover" style="background-image: url(${data['channel_icon']});"></div>
                     % else:
-                    <div id="poster-${sk}" class="dashboard-activity-poster-blur" style="background-image: url(pms_image_proxy?img=${data['channel_icon']}&width=300&height=300&fallback=cover&refresh=true);"></div>
+                    <div id="poster-${sk}" class="dashboard-activity-poster" style="background-image: url(pms_image_proxy?img=${data['channel_icon']}&width=300&height=300&opacity=60&background=282828&blur=3&fallback=cover&refresh=true);"></div>
                     <div id="poster-${sk}" class="dashboard-activity-cover" style="background-image: url(pms_image_proxy?img=${data['channel_icon']}&width=300&height=300&fallback=cover&refresh=true);"></div>
                     % endif
                 % endif

data/interfaces/default/home_stats.html

@@ -71,22 +71,21 @@ DOCUMENTATION :: END
 %>
 <div class="dashboard-stats-instance" id="stats-instance-${stat_id}" data-stat_id="${stat_id}">
     <div class="dashboard-stats-container">
-        <div class="dashboard-stats-background-overlay">
         % if stat_id in ('top_movies', 'popular_movies', 'top_tv', 'popular_tv', 'top_music', 'popular_music', 'last_watched'):
         % if row0['art']:
-            <div id="stats-background-${stat_id}" class="dashboard-stats-background" style="background-image: url(pms_image_proxy?img=${row0['art']}&width=500&height=280&fallback=art);"></div>
+        <div id="stats-background-${stat_id}" class="dashboard-stats-background" style="background-image: url(pms_image_proxy?img=${row0['art']}&width=500&height=280&opacity=40&background=282828&blur=3&fallback=art);">
         % else:
-            <div id="stats-background-${stat_id}" class="dashboard-stats-background" style="background-image: url(images/art.png);"></div>
+        <div id="stats-background-${stat_id}" class="dashboard-stats-background" style="background-image: url(images/art.png);">
         % endif
         % elif stat_id == 'top_platforms':
-            <div id="stats-background-${stat_id}" class="dashboard-stats-background platform-${row0['platform_name']} no-image"></div>
+        <div id="stats-background-${stat_id}" class="dashboard-stats-background platform-${row0['platform_name']}-rgba no-image">
         % else:
-            <div id="stats-background-${stat_id}" class="dashboard-stats-background flat"></div>
+        <div id="stats-background-${stat_id}" class="dashboard-stats-background flat">
         % endif
             % if stat_id in ('top_movies', 'popular_movies', 'top_tv', 'popular_tv', 'top_music', 'popular_music', 'last_watched'):
             <div class="dashboard-stats-poster-container hidden-xs">
                 % if stat_id in ('top_music', 'popular_music'):
-                <div id="stats-thumb-${stat_id}-bg" class="dashboard-stats-poster-blur" style="background-image: url(pms_image_proxy?img=${row0['thumb']}&width=300&height=300&fallback=cover);"></div>
+                <div id="stats-thumb-${stat_id}-bg" class="dashboard-stats-poster" style="background-image: url(pms_image_proxy?img=${row0['thumb']}&width=300&height=300&opacity=60&background=282828&blur=3&fallback=cover);"></div>
                 % endif
                 <% height, type = ('300', 'cover') if stat_id in ('top_music', 'popular_music') else ('450', 'poster') %>
                 <% href = 'info?rating_key={}'.format(row0['rating_key']) if row0['rating_key'] else '#' %>
@@ -200,7 +199,7 @@ DOCUMENTATION :: END
             }).addClass('platform-' + $(elem).data('platform'));
             $('#stats-background-' + stat_id).removeClass(function (index, className) {
                 return (className.match (/(^|\s)platform-\S+/g) || []).join(' ');
-            }).addClass('platform-' + $(elem).data('platform'));
+            }).addClass('platform-' + $(elem).data('platform') + '-rgba');
         } else {
             if (rating_key) {
                 href = 'info?rating_key=' + rating_key;
@@ -209,13 +208,13 @@ DOCUMENTATION :: END
             }
             $('#stats-thumb-url-' + stat_id).attr('href', href).prop('title', $(elem).data('title'));
             if (art) {
-                $('#stats-background-' + stat_id).css('background-image', 'url(pms_image_proxy?img=' + art + '&width=500&height=280&fallback=art)');
+                $('#stats-background-' + stat_id).css('background-image', 'url(pms_image_proxy?img=' + art + '&width=500&height=280&opacity=40&background=282828&blur=3&fallback=art)');
             } else {
                 $('#stats-background-' + stat_id).css('background-image', 'url(images/art.png)');
             }
             if (thumb) {
                 $('#stats-thumb-' + stat_id).css('background-image', 'url(pms_image_proxy?img=' + thumb + '&width=300&height=' + height + '&fallback=' + fallback + ')');
-                $('#stats-thumb-' + stat_id + '-bg').css('background-image', 'url(pms_image_proxy?img=' + thumb + '&width=300&height=' + height + '&fallback=' + fallback + ')');
+                $('#stats-thumb-' + stat_id + '-bg').css('background-image', 'url(pms_image_proxy?img=' + thumb + '&width=300&height=' + height + '&opacity=60&background=282828&blur=3&fallback=' + fallback + ')');
             } else {
                 $('#stats-thumb-' + stat_id).css('background-image', 'url(images/' + fallback + '.png)');
                 $('#stats-thumb-' + stat_id + '-bg').css('background-image', 'url(images/' + fallback + '.png)');

data/interfaces/default/index.html

@@ -10,16 +10,8 @@
     % if section == 'current_activity':
     <div class="row">
         <div class="col-md-12">
-            <div class="home-padded-header padded-header" id="current-activity-header">
-                <h3 class="pull-left"><span id="sessions-shortcut">Current Activity</span></h3>
-                <div class="button-bar">
-                    <div class="input-group pull-left" style="width: 1px; margin-right: 3px" id="activity-refresh-interval-selection">
-                        <span class="input-group-addon btn-dark inactive">Refresh Every</span>
-                        <input type="number" class="form-control number-input" name="activity-refresh-interval" id="activity-refresh-interval" value="${config['home_refresh_interval']}" min="2" data-default="2" data-toggle="tooltip" title="Min: 2 seconds" />
-                        <span class="input-group-addon btn-dark inactive">seconds</span>
-                    </div>
-                </div>
-                <div class="info-bar">
+            <div class="padded-header" id="current-activity-header">
+                <h3><span id="sessions-shortcut">Activity</span> &nbsp;&nbsp;
                     <small>
                         <span id="currentActivityHeader" style="display: none;">
                             Streams: <span id="currentActivityHeader-streams"></span> |
@@ -27,7 +19,7 @@
                             <span id="currentActivityHeader-bandwidth-tooltip" data-toggle="tooltip" title="Streaming Brain Estimate (Required Bandwidth)"><i class="fa fa-info-circle"></i></span>
                         </span>
                     </small>
-                </div>
+                </h3>
             </div>
             <div id="currentActivity">
                 <% from plexpy import PLEX_SERVER_UP %>
@@ -131,7 +123,7 @@
                         </label>
                     </div>
                     <div class="input-group pull-left" style="width: 1px;" id="recently-added-count-selection">
-                        <input type="number" class="form-control number-input" name="recently-added-count" id="recently-added-count" value="${config['home_stats_recently_added_count']}" min="1" max="100" data-default="50" data-toggle="tooltip" title="Min: 1 item<br>Max: 100 items" />
+                        <input type="number" class="form-control number-input" name="recently-added-count" id="recently-added-count" value="${config['home_stats_recently_added_count']}" min="1" max="50" data-default="50" data-toggle="tooltip" title="Min: 1 item<br>Max: 50 items" />
                         <span class="input-group-addon btn-dark inactive">items</span>
                     </div>
                 </div>
@@ -383,7 +375,7 @@
                         if (s.media_type === 'track') {
                             // Update if artist changed
                             if (s.grandparent_rating_key !== instance.data('grandparent_rating_key')) {
-                                $('#background-' + key).css('background-image', 'url(pms_image_proxy?img=' + s.art + '&width=500&height=280&fallback=art&refresh=true)');
+                                $('#background-' + key).css('background-image', 'url(pms_image_proxy?img=' + s.art + '&width=500&height=280&opacity=40&background=282828&blur=3&fallback=art&refresh=true)');
                                 $('#metadata-grandparent_title-' + key)
                                     .attr('href', 'info?rating_key=' + s.grandparent_rating_key)
                                     .attr('title', s.grandparent_title)
@@ -392,7 +384,7 @@
                             // Update cover if album changed
                             if (s.parent_rating_key !== instance.data('parent_rating_key')) {
                                 $('#poster-' + key).css('background-image', 'url(pms_image_proxy?img=' + s.parent_thumb + '&width=300&height=300&fallback=poster&refresh=true)');
-                                $('#poster-' + key + '-bg').css('background-image', 'url(pms_image_proxy?img=' + s.parent_thumb + '&width=300&height=300&fallback=poster&refresh=true)');
+                                $('#poster-' + key + '-bg').css('background-image', 'url(pms_image_proxy?img=' + s.parent_thumb + '&width=300&height=300&opacity=60&background=282828&blur=3&fallback=poster&refresh=true)');
                                 $('#poster-url-' + key)
                                     .attr('href', 'info?rating_key=' + s.parent_rating_key)
                                     .attr('title', s.parent_title);
@@ -591,16 +583,11 @@
     }
 
     getCurrentActivity();
-
-    function refreshActivity(seconds) {
-        return setInterval(function () {
+    setInterval(function () {
         if (!(create_instances.length) && activity_ready) {
             getCurrentActivity();
         }
-        }, seconds * 1000);
-    }
-    var refresh_interval = $('#activity-refresh-interval').val();
-    var activityRefresh = refreshActivity(refresh_interval);
+    }, ${config['home_refresh_interval'] * 1000});
 
     setInterval(function(){
         $('.progress_time_offset').each(function () {
@@ -696,16 +683,6 @@
            window.open(sessions_url, '_blank');
         });
     });
-
-    $('#activity-refresh-interval').change(function () {
-        forceMinMax($(this));
-        clearInterval(activityRefresh);
-        refresh_interval = $(this).val();
-        activityRefresh = refreshActivity(refresh_interval);
-        $.post('set_home_stats_config', { refresh_interval: refresh_interval });
-    });
-
-    $('#activity-refresh-interval').tooltip({ container: 'body', placement: 'top', html: true });
     % endif
 </script>
 % endif

data/interfaces/default/js/tables/newsletter_logs.js

@@ -0,0 +1,146 @@
+newsletter_log_table_options = {
+    "destroy": true,
+    "serverSide": true,
+    "processing": false,
+    "pagingType": "full_numbers",
+    "order": [ 0, 'desc'],
+    "pageLength": 50,
+    "stateSave": true,
+    "language": {
+                "search":"Search: ",
+                "lengthMenu": "Show _MENU_ lines per page",
+                "emptyTable": "No log information available",
+                "info" :"Showing _START_ to _END_ of _TOTAL_ lines",
+                "infoEmpty": "Showing 0 to 0 of 0 lines",
+                "infoFiltered": "(filtered from _MAX_ total lines)",
+                "loadingRecords": '<i class="fa fa-refresh fa-spin"></i> Loading items...</div>'
+    },
+    "autoWidth": false,
+    "scrollX": true,
+    "columnDefs": [
+        {
+            "targets": [0],
+            "data": "timestamp",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData !== '') {
+                    $(td).html(moment(cellData, "X").format('YYYY-MM-DD HH:mm:ss'));
+                }
+            },
+            "width": "10%",
+            "className": "no-wrap"
+        },
+        {
+            "targets": [1],
+            "data": "newsletter_id",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData !== '') {
+                    $(td).html(cellData);
+                }
+            },
+            "width": "5%",
+            "className": "no-wrap"
+        },
+        {
+            "targets": [2],
+            "data": "agent_name",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData !== '') {
+                    $(td).html(cellData);
+                }
+            },
+            "width": "5%",
+            "className": "no-wrap"
+        },
+        {
+            "targets": [3],
+            "data": "notify_action",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData !== '') {
+                    $(td).html(cellData);
+                }
+            },
+            "width": "5%",
+            "className": "no-wrap"
+        },
+        {
+            "targets": [4],
+            "data": "subject_text",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData !== '') {
+                    $(td).html(cellData);
+                }
+            },
+            "width": "23%"
+        },
+        {
+            "targets": [5],
+            "data": "body_text",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData !== '') {
+                    $(td).html(cellData);
+                }
+            },
+            "width": "35%"
+        },
+        {
+            "targets": [6],
+            "data": "start_date",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData !== '') {
+                    $(td).html(cellData);
+                }
+            },
+            "width": "5%"
+        },
+        {
+            "targets": [7],
+            "data": "end_date",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData !== '') {
+                    $(td).html(cellData);
+                }
+            },
+            "width": "5%"
+        },
+        {
+            "targets": [8],
+            "data": "uuid",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData !== '') {
+                    $(td).html('<a href="newsletter/' + rowData['uuid'] + '" target="_blank">' + cellData + '</a>');
+                }
+            },
+            "width": "5%"
+        },
+        {
+            "targets": [9],
+            "data": "success",
+            "createdCell": function (td, cellData, rowData, row, col) {
+                if (cellData === 1) {
+                    $(td).html('<span class="success-tooltip" data-toggle="tooltip" title="Newsletter Sent"><i class="fa fa-lg fa-fw fa-check"></i></span>');
+                } else {
+                    $(td).html('<span class="success-tooltip" data-toggle="tooltip" title="Newsletter Failed"><i class="fa fa-lg fa-fw fa-times"></i></span>');
+                }
+            },
+            "searchable": false,
+            "orderable": false,
+            "className": "no-wrap",
+            "width": "2%"
+        },
+    ],
+    "drawCallback": function (settings) {
+        // Jump to top of page
+        //$('html,body').scrollTop(0);
+        $('#ajaxMsg').fadeOut();
+
+        // Create the tooltips.
+        $('body').tooltip({
+            selector: '[data-toggle="tooltip"]',
+            container: 'body'
+        });
+    },
+    "preDrawCallback": function(settings) {
+        var msg = "<i class='fa fa-refresh fa-spin'></i>&nbspFetching rows...";
+        showMsg(msg, false, false, 0)
+    }
+};

data/interfaces/default/js/tables/notification_logs.js

@@ -86,7 +86,7 @@ notification_log_table_options = {
             "targets": [6],
             "data": "success",
             "createdCell": function (td, cellData, rowData, row, col) {
-                if (cellData == 1) {
+                if (cellData === 1) {
                     $(td).html('<span class="success-tooltip" data-toggle="tooltip" title="Notification Sent"><i class="fa fa-lg fa-fw fa-check"></i></span>');
                 } else {
                     $(td).html('<span class="success-tooltip" data-toggle="tooltip" title="Notification Failed"><i class="fa fa-lg fa-fw fa-times"></i></span>');
@@ -113,4 +113,4 @@ notification_log_table_options = {
         var msg = "<i class='fa fa-refresh fa-spin'></i>&nbspFetching rows...";
         showMsg(msg, false, false, 0)
     }
-}
+};

data/interfaces/default/library_stats.html

@@ -35,8 +35,7 @@ DOCUMENTATION :: END
 % if section_type in data:
 <div class="dashboard-stats-instance" id="library-stats-instance-${section_type}" data-section_type="${section_type}">
     <div class="dashboard-stats-container">
-        <div class="dashboard-stats-background-overlay">
-            <div id="library-stats-background-${section_type}" class="dashboard-stats-background" style="background-image: url(pms_image_proxy?img=/:/resources/${section_type}-fanart.jpg&width=500&height=280&fallback=art);"></div>
+        <div id="library-stats-background-${section_type}" class="dashboard-stats-background" style="background-image: url(pms_image_proxy?img=/:/resources/${section_type}-fanart.jpg&width=500&height=280&opacity=40&background=282828&blur=3&fallback=art);">
             <div id="library-stats-thumb-${section_type}" class="dashboard-stats-flat svg-icon library-${section_type} hidden-xs"></div>
             <div class="dashboard-stats-info-container">
                 <div id="library-stats-title-${section_type}" class="dashboard-stats-info-title">

data/interfaces/default/login.html

@@ -85,7 +85,7 @@
             dataType: 'json',
             statusCode: {
                 200: function() {
-                   window.location = "${http_root}";
+                   window.location = "${redirect_uri or http_root}";
                 },
                 401: function() {
                    $('#incorrect-login').show();

data/interfaces/default/logs.html

@@ -50,6 +50,7 @@
             <button class="btn btn-dark" id="download-plexscannerlog" style="display: none;"><i class="fa fa-download"></i> Download logs</button>
             <button class="btn btn-dark" id="clear-logs"><i class="fa fa-trash-o"></i> Clear logs</button>
             <button class="btn btn-dark" id="clear-notify-logs" style="display: none;"><i class="fa fa-trash-o"></i> Clear logs</button>
+            <button class="btn btn-dark" id="clear-newsletter-logs" style="display: none;"><i class="fa fa-trash-o"></i> Clear logs</button>
             <button class="btn btn-dark" id="clear-login-logs" style="display: none;"><i class="fa fa-trash-o"></i> Clear logs</button>
         </div>
     </div>
@@ -62,6 +63,7 @@
                 <li role="presentation"><a id="plex-scanner-logs-btn" href="#tabs-plex_scanner_log" aria-controls="tabs-plex_scanner_log" role="tab" data-toggle="tab">Plex Media Scanner Logs</a></li>
                 <li role="presentation"><a id="plex-websocket-logs-btn" href="#tabs-plex_websocket_log" aria-controls="tabs-plex_websocket_log" role="tab" data-toggle="tab">Plex Websocket Logs</a></li>
                 <li role="presentation"><a id="notification-logs-btn" href="#tabs-notification_log" aria-controls="tabs-notification_log" role="tab" data-toggle="tab">Notification Logs</a></li>
+                <li role="presentation"><a id="newsletter-logs-btn" href="#tabs-newsletter_log" aria-controls="tabs-newsletter_log" role="tab" data-toggle="tab">Newsletter Logs</a></li>
                 <li role="presentation"><a id="login-logs-btn" href="#tabs-login_log" aria-controls="tabs-login_log" role="tab" data-toggle="tab">Login Logs</a></li>
             </ul>
             <div class="tab-content">
@@ -141,6 +143,25 @@
                         <tbody></tbody>
                     </table>
                 </div>
+                <div role="tabpanel" class="tab-pane" id="tabs-newsletter_log">
+                    <table class="display" id="newsletter_log_table" width="100%">
+                        <thead>
+                            <tr>
+                                <th align="left" id="newsletter_timestamp">Timestamp</th>
+                                <th align="left" id="newsletter_newsletter_id">Newsletter ID</th>
+                                <th align="left" id="newsletter_agent_name">Agent</th>
+                                <th align="left" id="newsletter_notify_action">Action</th>
+                                <th align="left" id="newsletter_subject_text">Subject Text</th>
+                                <th align="left" id="newsletter_body_text">Body Text</th>
+                                <th align="left" id="newsletter_start_date">Start Date</th>
+                                <th align="left" id="newsletter_end_date">End Date</th>
+                                <th align="left" id="newsletter_uuid">UUID</th>
+                                <th align="left" id="newsletter_success"></th>
+                            </tr>
+                        </thead>
+                        <tbody></tbody>
+                    </table>
+                </div>
                 <div role="tabpanel" class="tab-pane" id="tabs-login_log">
                     <table class="display login_log_table" id="login_log_table" width="100%">
                         <thead>
@@ -191,6 +212,7 @@
 <script src="${http_root}js/tables/logs.js${cache_param}"></script>
 <script src="${http_root}js/tables/plex_logs.js${cache_param}"></script>
 <script src="${http_root}js/tables/notification_logs.js${cache_param}"></script>
+<script src="${http_root}js/tables/newsletter_logs.js${cache_param}"></script>
 <script src="${http_root}js/tables/login_logs.js${cache_param}"></script>
 <script>
 
@@ -278,6 +300,18 @@
             notification_log_table = $('#notification_log_table').DataTable(notification_log_table_options);
         }
 
+        function loadNewsletterLogs() {
+            newsletter_log_table_options.ajax = {
+                url: "get_newsletter_log",
+                data: function (d) {
+                    return {
+                        json_data: JSON.stringify(d)
+                    };
+                }
+            };
+            newsletter_log_table = $('#newsletter_log_table').DataTable(newsletter_log_table_options);
+        }
+
         function loadLoginLogs() {
             login_log_table_options.pageLength = 50;
             login_log_table_options.ajax = {
@@ -300,6 +334,7 @@
             $("#download-plexserverlog").hide();
             $("#download-plexscannerlog").hide();
             $("#clear-notify-logs").hide();
+            $("#clear-newsletter-logs").hide();
             $("#clear-login-logs").hide();
             loadtautullilogs('tautulli', selected_log_level);
             clearSearchButton('tautulli_log_table', log_table);
@@ -313,6 +348,7 @@
             $("#download-plexserverlog").hide();
             $("#download-plexscannerlog").hide();
             $("#clear-notify-logs").hide();
+             $("#clear-newsletter-logs").hide();
            $("#clear-login-logs").hide();
             loadtautullilogs('tautulli_api', selected_log_level);
             clearSearchButton('tautulli_api_log_table', log_table);
@@ -326,6 +362,7 @@
             $("#download-plexserverlog").hide();
             $("#download-plexscannerlog").hide();
             $("#clear-notify-logs").hide();
+            $("#clear-newsletter-logs").hide();
             $("#clear-login-logs").hide();
             loadtautullilogs('plex_websocket', selected_log_level);
             clearSearchButton('plex_websocket_log_table', log_table);
@@ -339,6 +376,7 @@
             $("#download-plexserverlog").show();
             $("#download-plexscannerlog").hide();
             $("#clear-notify-logs").hide();
+            $("#clear-newsletter-logs").hide();
             $("#clear-login-logs").hide();
             loadPlexLogs();
             clearSearchButton('plex_log_table', plex_log_table);
@@ -352,6 +390,7 @@
             $("#download-plexserverlog").hide();
             $("#download-plexscannerlog").show();
             $("#clear-notify-logs").hide();
+            $("#clear-newsletter-logs").hide();
             $("#clear-login-logs").hide();
             loadPlexScannerLogs();
             clearSearchButton('plex_scanner_log_table', plex_scanner_log_table);
@@ -365,11 +404,26 @@
             $("#download-plexserverlog").hide();
             $("#download-plexscannerlog").hide();
             $("#clear-notify-logs").show();
+            $("#clear-newsletter-logs").hide();
             $("#clear-login-logs").hide();
             loadNotificationLogs();
             clearSearchButton('notification_log_table', notification_log_table);
         });
 
+        $("#newsletter-logs-btn").click(function () {
+            $("#tautulli-log-levels").hide();
+            $("#plex-log-levels").hide();
+            $("#clear-logs").hide();
+            $("#download-tautullilog").hide();
+            $("#download-plexserverlog").hide();
+            $("#download-plexscannerlog").hide();
+            $("#clear-notify-logs").hide();
+            $("#clear-newsletter-logs").show();
+            $("#clear-login-logs").hide();
+            loadNewsletterLogs();
+            clearSearchButton('newsletter_log_table', newsletter_log_table);
+        });
+
         $("#login-logs-btn").click(function () {
             $("#tautulli-log-levels").hide();
             $("#plex-log-levels").hide();
@@ -378,6 +432,7 @@
             $("#download-plexserverlog").hide();
             $("#download-plexscannerlog").hide();
             $("#clear-notify-logs").hide();
+            $("#clear-newsletter-logs").hide();
             $("#clear-login-logs").show();
             loadLoginLogs();
             clearSearchButton('login_log_table', notification_log_table);
@@ -446,6 +501,27 @@
             });
         });
 
+        $("#clear-newsletter-logs").click(function () {
+            $("#confirm-message").text("Are you sure you want to clear the Tautulli Newsletter Logs?");
+            $('#confirm-modal').modal();
+            $('#confirm-modal').one('click', '#confirm-button', function () {
+                $.ajax({
+                    url: 'delete_newsletter_log',
+                    type: 'POST',
+                    complete: function (xhr, status) {
+                        result = $.parseJSON(xhr.responseText);
+                        msg = result.message;
+                        if (result.result === 'success') {
+                            showMsg('<i class="fa fa-check"></i> ' + msg, false, true, 5000)
+                        } else {
+                            showMsg('<i class="fa fa-times"></i> ' + msg, false, true, 5000, true)
+                        }
+                        newsletter_log_table.draw();
+                    }
+                });
+            });
+        });
+
         $("#clear-login-logs").click(function () {
             $("#confirm-message").text("Are you sure you want to clear the Tautulli Login Logs?");
             $('#confirm-modal').modal();

data/interfaces/default/newsletter_config.html

@@ -0,0 +1,723 @@
+% if newsletter:
+<%!
+    import json
+    from plexpy import notifiers
+    from plexpy.helpers import anon_url, checked
+
+    all_notifiers = sorted(notifiers.get_notifiers(), key=lambda k: (k['agent_label'].lower(), k['friendly_name'], k['id']))
+    email_notifiers = [n for n in all_notifiers if n['agent_name'] == 'email']
+    email_notifiers = [{'id': 0, 'agent_label': 'New Email Configuration', 'friendly_name': ''}] + email_notifiers
+    other_notifiers = [{'id': 0, 'agent_label': 'Select a Notification Agent', 'friendly_name': ''}] + all_notifiers
+%>
+<div class="modal-dialog" role="document">
+    <div class="modal-content">
+        <div class="modal-header">
+            <button type="button" class="close" data-dismiss="modal" aria-hidden="true"><i class="fa fa-remove"></i></button>
+            <h4 class="modal-title" id="newsletter-config-modal-header">${newsletter['agent_label']} Newsletter Settings &nbsp;<small><span class="newsletter_id">(Newsletter ID: ${newsletter['id']})</span></small></h4>
+        </div>
+        <div class="modal-body">
+            <div class="container-fluid">
+                <div class="row">
+                    <ul class="nav nav-tabs list-unstyled" role="tablist">
+                        <li role="presentation" class="active"><a href="#tabs-newsletter_config" aria-controls="tabs-newsletter_config" role="tab" data-toggle="tab">Configuration</a></li>
+                        <li role="presentation"><a href="#tabs-newsletter_agent" aria-controls="tabs-newsletter_agent" role="tab" data-toggle="tab">Notification Agent</a></li>
+                        <li role="presentation"><a href="#tabs-newsletter_text" aria-controls="tabs-newsletter_text" role="tab" data-toggle="tab">Newsletter Text</a></li>
+                        <li role="presentation"><a href="#tabs-test_newsletter" aria-controls="tabs-test_newsletter" role="tab" data-toggle="tab">Test Newsletter</a></li>
+                    </ul>
+                </div>
+                <form action="set_newsletter_config" method="post" class="form" id="set_newsletter_config" data-parsley-validate>
+                    <div class="tab-content">
+                        <div role="tabpanel" class="tab-pane active" id="tabs-newsletter_config">
+                            <div class="row">
+                                <div class="col-md-12">
+                                    <div class="checkbox" style="margin-bottom: 20px;">
+                                        <label>
+                                            <input type="checkbox" data-id="active_value" class="checkboxes" value="1" ${checked(newsletter['active'])}> Enable the Newsletter
+                                        </label>
+                                        <input type="hidden" id="active_value" name="active" value="${newsletter['active']}">
+                                    </div>
+                                    <div class="form-group">
+                                        <label for="custom_cron">Schedule</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <select class="form-control" id="custom_cron" name="newsletter_config_custom_cron">
+                                                    <option value="0" ${'selected' if newsletter['config']['custom_cron'] == 0 else ''}>Simple</option>
+                                                    <option value="1" ${'selected' if newsletter['config']['custom_cron'] == 1 else ''}>Custom</option>
+                                                </select>
+                                                <input type="text" id="cron_value" name="cron" value="${newsletter['cron']}" />
+                                                <div id="cron-widget"></div>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">
+                                            <span id="simple_cron_message">Set the schedule for the newsletter.</span>
+                                            <span id="custom_cron_message">Set the schedule for the newsletter using a <a href="${anon_url('https://crontab.guru')}" target="_blank">custom crontab</a>. Only standard cron values are valid.</span>
+                                        </p>
+                                    </div>
+                                </div>
+                                <div class="col-md-12" style="padding-top: 10px; border-top: 1px solid #444;">
+                                    <input type="hidden" id="newsletter_id" name="newsletter_id" value="${newsletter['id']}" />
+                                    <input type="hidden" id="agent_id" name="agent_id" value="${newsletter['agent_id']}" />
+                                    % for item in newsletter['config_options']:
+                                    % if item['input_type'] == 'help':
+                                    <div class="form-group">
+                                        <label>${item['label']}</label>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'text' or item['input_type'] == 'password':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <input type="${item['input_type']}" class="form-control" id="${item['name']}" name="${item['name']}" value="${item['value']}" size="30" ${'readonly' if item.get('readonly') else ''}>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'number':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-3">
+                                                <input type="${item['input_type']}" class="form-control" id="${item['name']}" name="${item['name']}" value="${item['value']}" size="30">
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'button':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <input type="button" class="btn btn-bright" id="${item['name']}" name="${item['name']}" value="${item['value']}">
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'checkbox':
+                                    <div class="checkbox">
+                                        <label>
+                                            <input type="checkbox" data-id="${item['name']}" class="checkboxes" value="1" ${checked(item['value'])}> ${item['label']}
+                                        </label>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                        <input type="hidden" id="${item['name']}" name="${item['name']}" value="${item['value']}">
+                                    </div>
+                                    % elif item['input_type'] == 'select':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <select class="form-control" id="${item['name']}" name="${item['name']}">
+                                                    % for key, value in sorted(item['select_options'].iteritems()):
+                                                    % if key == item['value']:
+                                                    <option value="${key}" selected>${value}</option>
+                                                    % else:
+                                                    <option value="${key}">${value}</option>
+                                                    % endif
+                                                    % endfor
+                                                </select>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'selectize':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <select class="form-control" id="${item['name']}" name="${item['name']}">
+                                                    <option value="select-all">Select All</option>
+                                                    <option value="remove-all">Remove All</option>
+                                                    % if isinstance(item['select_options'], dict):
+                                                    % for section, options in item['select_options'].iteritems():
+                                                    <optgroup label="${section}">
+                                                        % for option in sorted(options, key=lambda x: x['text'].lower()):
+                                                            <option value="${option['value']}">${option['text']}</option>
+                                                        % endfor
+                                                    </optgroup>
+                                                    % endfor
+                                                    % else:
+                                                    <option value="border-all"></option>
+                                                    % for option in sorted(item['select_options'], key=lambda x: x['text'].lower()):
+                                                        <option value="${option['value']}">${option['text']}</option>
+                                                    % endfor
+                                                    % endif
+                                                </select>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % endif
+                                    % endfor
+                                </div>
+                                <div class="col-md-12" style="margin-top: 10px; padding-top: 10px; border-top: 1px solid #444;">
+                                    <div class="form-group">
+                                        <label for="friendly_name">Description</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <input type="text" class="form-control" id="friendly_name" name="friendly_name" value="${newsletter['friendly_name']}" size="30">
+                                            </div>
+                                        </div>
+                                        <p class="help-block">Optional: Enter a description to help identify this newsletter in the newsletters list.</p>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+                        <div role="tabpanel" class="tab-pane" id="tabs-newsletter_agent">
+                            <div class="row">
+                                <div class="col-md-12">
+                                    <div class="checkbox">
+                                        <label>
+                                            <input type="checkbox" id="newsletter_config_formatted_checkbox" data-id="newsletter_config_formatted" class="checkboxes" value="1" ${checked(newsletter['config']['formatted'])}> Send newsletter as an HTML formatted Email
+                                        </label>
+                                        <p class="help-block">Enable to send the newsletter as an HTML formatted Email. Disable to only send a subject and body message to a different notification agent.</p>
+                                        <input type="hidden" id="newsletter_config_formatted" name="newsletter_config_formatted" value="${newsletter['config']['formatted']}">
+                                    </div>
+                                    <div class="form-group" id="email_notifier_select">
+                                        <label for="newsletter_email_notifier_id">Email Notification Agent</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <select class="form-control" id="newsletter_email_notifier_id" name="newsletter_email_notifier_id">
+                                                    % for notifier in email_notifiers:
+                                                    <% selected = 'selected' if notifier['id'] == newsletter['email_config']['notifier_id'] else '' %>
+                                                    % if notifier['friendly_name']:
+                                                    <option value="${notifier['id']}" ${selected}>${notifier['agent_label']} (${notifier['id']} - ${notifier['friendly_name']})</option>
+                                                    % elif notifier['id']:
+                                                    <option value="${notifier['id']}" ${selected}>${notifier['agent_label']} (${notifier['id']})</option>
+                                                    % else:
+                                                    <option value="${notifier['id']}" ${selected}>${notifier['agent_label']}</option>
+                                                    % endif
+                                                    % endfor
+                                                </select>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">
+                                            Select an existing Email notification agent or enter a new configuration below.<br>
+                                            Note: Make sure HTML support is enabled for the Email notification agent.
+                                        </p>
+                                    </div>
+                                    <div class="form-group" id="other_notifier_select">
+                                        <label for="newsletter_config_notifier_id">Notification Agent</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <select class="form-control" id="newsletter_config_notifier_id" name="newsletter_config_notifier_id">
+                                                    % for notifier in other_notifiers:
+                                                    <% selected = 'selected' if notifier['id'] == newsletter['config']['notifier_id'] else '' %>
+                                                    % if notifier['friendly_name']:
+                                                    <option value="${notifier['id']}" ${selected}>${notifier['agent_label']} (${notifier['id']} - ${notifier['friendly_name']})</option>
+                                                    % elif notifier['id']:
+                                                    <option value="${notifier['id']}" ${selected}>${notifier['agent_label']} (${notifier['id']})</option>
+                                                    % else:
+                                                    <option value="${notifier['id']}" ${selected}>${notifier['agent_label']}</option>
+                                                    % endif
+                                                    % endfor
+                                                </select>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">
+                                            Select an existing notification agent where the subject and body text will be sent.<br>
+                                            Note: Self-hosted newsletters must be enabled under <a data-tab-destination="tabs-notifications" data-dismiss="modal" data-target="#newsletter_self_hosted">Newsletters</a> to include a link to the newsletter.
+                                        </p>
+                                    </div>
+                                </div>
+                                <div id="newsletter-email-config" class="col-md-12" style="padding-top: 10px; border-top: 1px solid #444;">
+                                    % for item in newsletter['email_config_options']:
+                                    % if item['input_type'] == 'help':
+                                    <div class="form-group">
+                                        <label>${item['label']}</label>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'text' or item['input_type'] == 'password':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <input type="${item['input_type']}" class="form-control" id="${item['name']}" name="${item['name']}" value="${item['value']}" size="30" ${'readonly' if item.get('readonly') else ''}>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'number':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-3">
+                                                <input type="${item['input_type']}" class="form-control" id="${item['name']}" name="${item['name']}" value="${item['value']}" size="30">
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'button':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <input type="button" class="btn btn-bright" id="${item['name']}" name="${item['name']}" value="${item['value']}">
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'checkbox' and item['name'] != 'newsletter_email_html_support':
+                                    <div class="checkbox">
+                                        <label>
+                                            <input type="checkbox" data-id="${item['name']}" class="checkboxes" value="1" ${checked(item['value'])}> ${item['label']}
+                                        </label>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                        <input type="hidden" id="${item['name']}" name="${item['name']}" value="${item['value']}">
+                                    </div>
+                                    % elif item['input_type'] == 'select':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <select class="form-control" id="${item['name']}" name="${item['name']}">
+                                                    % for key, value in sorted(item['select_options'].iteritems()):
+                                                    % if key == item['value']:
+                                                    <option value="${key}" selected>${value}</option>
+                                                    % else:
+                                                    <option value="${key}">${value}</option>
+                                                    % endif
+                                                    % endfor
+                                                </select>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % elif item['input_type'] == 'selectize':
+                                    <div class="form-group">
+                                        <label for="${item['name']}">${item['label']}</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <select class="form-control" id="${item['name']}" name="${item['name']}">
+                                                    <option value="select-all">Select All</option>
+                                                    <option value="remove-all">Remove All</option>
+                                                    % if isinstance(item['select_options'], dict):
+                                                    % for section, options in item['select_options'].iteritems():
+                                                    <optgroup label="${section}">
+                                                        % for option in sorted(options, key=lambda x: x['text'].lower()):
+                                                            <option value="${option['value']}">${option['text']}</option>
+                                                        % endfor
+                                                    </optgroup>
+                                                    % endfor
+                                                    % else:
+                                                    <option value="border-all"></option>
+                                                    % for option in sorted(item['select_options'], key=lambda x: x['text'].lower()):
+                                                        <option value="${option['value']}">${option['text']}</option>
+                                                    % endfor
+                                                    % endif
+                                                </select>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">${item['description'] | n}</p>
+                                    </div>
+                                    % endif
+                                    % endfor
+                                    <input type="hidden" id="newsletter_email_html_support" name="newsletter_email_html_support" value="1">
+                                </div>
+                            </div>
+                        </div>
+                        <div role="tabpanel" class="tab-pane" id="tabs-newsletter_text">
+                            <label>Newsletter Text</label>
+                            <p class="help-block">
+                                Set the custom formatted text for each type of notification.
+                                <a href="#newsletter-text-sub-modal" data-toggle="modal">Click here</a> for a list of available parameters which can be used.
+                            </p>
+                            <p class="help-block">
+                                You can also add text modifiers to change the case or slice parameters with a list of items.
+                                <a href="#notify-text-modifiers-modal" data-toggle="modal">Click here</a> to view usage information.
+                            </p>
+                            <div class="row">
+                                <div class="col-md-12">
+                                    <div class="form-group">
+                                        <label for="subject">Subject</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <input type="text" class="form-control" id="subject" name="subject" value="${newsletter['subject']}" size="30">
+                                            </div>
+                                        </div>
+                                        <p class="help-block">
+                                            Enter a custom subject line for the newsletter. Leave blank for default.
+                                        </p>
+                                    </div>
+                                    <div class="form-group" id="newsletter_body">
+                                        <label for="body">Body</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <textarea class="form-control"  id="body" name="body" data-autoresize>${newsletter['body']}</textarea>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">
+                                            Enter a custom body line for the newsletter notification. Leave blank for default.
+                                        </p>
+                                    </div>
+                                    <div class="form-group">
+                                        <label for="message">Message</label>
+                                        <div class="row">
+                                            <div class="col-md-12">
+                                                <textarea class="form-control"  id="message" name="message" data-autoresize>${newsletter['message']}</textarea>
+                                            </div>
+                                        </div>
+                                        <p class="help-block">
+                                            Enter a custom message to include on the newsletter.
+                                        </p>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+                        <div role="tabpanel" class="tab-pane" id="tabs-test_newsletter">
+                            <label>Preview Newsletter</label>
+                            <p class="help-block">
+                                Preview the ${newsletter['agent_label']} newsletter.
+                            </p>
+                            <div class="form-group">
+                                <div class="row">
+                                    <div class="col-md-12">
+                                        <input type="button" class="btn btn-bright" id="preview_newsletter" name="preview_newsletter" value="Preview ${newsletter['agent_label']} Newsletter">
+                                    </div>
+                                </div>
+                            </div>
+                            <label>Test Newsletter</label>
+                            <p class="help-block">
+                                Test if the ${newsletter['agent_label']} newsletter is working. Check the <a href="logs">logs</a> for troubleshooting.
+                            </p>
+                            <p class="help-block">
+                                Warning: This will send an actual newsletter to your notification agent!
+                            </p>
+                            <div class="form-group">
+                                <div class="row">
+                                    <div class="col-md-12">
+                                        <input type="button" class="btn btn-bright" id="test_newsletter" name="test_newsletter" value="Test ${newsletter['agent_label']} Newsletter">
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </form>
+            </div>
+        </div>
+        <div class="modal-footer">
+            <input type="button" id="delete-newsletter-item" class="btn btn-danger btn-edit" style="float:left;" value="Delete">
+            <input type="button" id="duplicate-newsletter-item" class="btn btn-dark btn-edit" style="float:left;" value="Duplicate">
+            <input type="button" id="save-newsletter-item" class="btn btn-bright" value="Save">
+        </div>
+    </div>
+</div>
+
+<script src="${http_root}js/jquery-cron-min.js"></script>
+<script>
+
+    $('#newsletter-config-modal').unbind('hidden.bs.modal');
+
+    var cron_widget = $('#cron-widget').cron({
+        initial: '0 0 * * 0',
+        classes: 'form-control cron-select',
+        onChange: function() {
+            $("#cron_value").val($(this).cron('value'));
+        }
+    });
+
+    if (${newsletter['config']['custom_cron']}) {
+        $('#cron_value').val('${newsletter['cron']}');
+    } else {
+        try {
+            cron_widget.cron('value', '${newsletter['cron']}');
+        } catch (e) {}
+    }
+
+    function toggleCustomCron() {
+       if ($('#custom_cron').val() === '1'){
+           $('#cron-widget').hide();
+           $('#cron_value').show();
+           $('#simple_cron_message').hide();
+           $('#custom_cron_message').show();
+       } else {
+           $('#cron-widget').show();
+           $('#cron_value').hide();
+           $('#simple_cron_message').show();
+           $('#custom_cron_message').hide();
+       }
+    }
+    toggleCustomCron();
+
+    $('#custom_cron').change(function () {
+        toggleCustomCron();
+    });
+
+    var $incl_libraries = $('#newsletter_config_incl_libraries').selectize({
+        plugins: ['remove_button'],
+        maxItems: null,
+        render: {
+            option: function(item) {
+                if (item.value.endsWith('-all')) {
+                    return '<div class="' + item.value + '">' + item.text + '</div>'
+                }
+                return '<div>' + item.text + '</div>';
+            }
+        },
+        onItemAdd: function(value) {
+            if (value === 'select-all') {
+                var all_keys = $.map(this.options, function(option){
+                    return option.value.endsWith('-all') ? null : option.value;
+                });
+                this.setValue(all_keys);
+            } else if (value === 'remove-all') {
+                this.clear();
+                this.refreshOptions();
+                this.positionDropdown();
+            }
+        }
+    });
+    var incl_libraries = $incl_libraries[0].selectize;
+    incl_libraries.setValue(${json.dumps(next((c['value'] for c in newsletter['config_options'] if c['name'] == 'newsletter_config_incl_libraries'), [])) | n});
+
+    function toggleEmailSelect () {
+        if ($('#newsletter_config_formatted_checkbox').is(':checked')) {
+            $('#newsletter_body').hide();
+            $('#email_notifier_select').show();
+            $('#other_notifier_select').hide();
+            toggleNewEmailConfig();
+        } else {
+            $('#newsletter_body').show();
+            $('#email_notifier_select').hide();
+            $('#other_notifier_select').show();
+            $('#newsletter-email-config').hide();
+        }
+    }
+    toggleEmailSelect();
+
+    $('#newsletter_config_formatted_checkbox').change(function () {
+        toggleEmailSelect();
+    });
+
+    function toggleNewEmailConfig () {
+        if ($('#newsletter_config_formatted_checkbox').is(':checked') && $('#newsletter_email_notifier_id').val() === '0') {
+            $('#newsletter-email-config').show();
+        } else {
+            $('#newsletter-email-config').hide();
+        }
+    }
+    toggleNewEmailConfig();
+
+    $('#newsletter_email_notifier_id').change(function () {
+        toggleNewEmailConfig();
+    });
+
+    var REGEX_EMAIL = '([a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*@' +
+                      '(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)';
+    var $email_selectors = $('#newsletter_email_to, #newsletter_email_cc, #newsletter_email_bcc').selectize({
+        plugins: ['remove_button'],
+        maxItems: null,
+        render: {
+            item: function(item, escape) {
+                return '<div>' +
+                    (item.text ? '<span class="item-text">' + escape(item.text) + '</span>' : '') +
+                    (item.value ? '<span class="item-value">' + escape(item.value) + '</span>' : '') +
+                '</div>';
+            },
+            option: function(item, escape) {
+                var label = item.text || item.value;
+                var caption = item.text ? item.value : null;
+                if (item.value.endsWith('-all')) {
+                    return '<div class="' + item.value + '">' + escape(label) + '</div>'
+                }
+                return '<div>' +
+                    escape(label) +
+                    (caption ? '<span class="caption">' + escape(caption) + '</span>' : '') +
+                '</div>';
+            }
+        },
+        onItemAdd: function(value) {
+            if (value === 'select-all') {
+                var all_keys = $.map(this.options, function(option){
+                    return option.value.endsWith('-all') ? null : option.value;
+                });
+                this.setValue(all_keys);
+            } else if (value === 'remove-all') {
+                this.clear();
+                this.refreshOptions();
+                this.positionDropdown();
+            }
+        },
+        createFilter: function(input) {
+            var match, regex;
+
+            // email@address.com
+            regex = new RegExp('^' + REGEX_EMAIL + '$', 'i');
+            match = input.match(regex);
+            if (match) return !this.options.hasOwnProperty(match[0]);
+
+            // user <email@address.com>
+            regex = new RegExp('^([^<]*)\<' + REGEX_EMAIL + '\>$', 'i');
+            match = input.match(regex);
+            if (match) return !this.options.hasOwnProperty(match[2]);
+
+            return false;
+        },
+        create: function(input) {
+            if ((new RegExp('^' + REGEX_EMAIL + '$', 'i')).test(input)) {
+                return {value: input};
+            }
+            var match = input.match(new RegExp('^([^<]*)\<' + REGEX_EMAIL + '\>$', 'i'));
+            if (match) {
+                return {
+                    value : match[2],
+                    text  : $.trim(match[1])
+                };
+            }
+            return false;
+        }
+    });
+    var email_to = $email_selectors[0].selectize;
+    var email_cc = $email_selectors[1].selectize;
+    var email_bcc = $email_selectors[2].selectize;
+    email_to.setValue(${json.dumps(next((c['value'] for c in newsletter['email_config_options'] if c['name'] == 'newsletter_email_to'), [])) | n});
+    email_cc.setValue(${json.dumps(next((c['value'] for c in newsletter['email_config_options'] if c['name'] == 'newsletter_email_cc'), [])) | n});
+    email_bcc.setValue(${json.dumps(next((c['value'] for c in newsletter['email_config_options'] if c['name'] == 'newsletter_email_bcc'), [])) | n});
+
+    function reloadModal() {
+        $.ajax({
+            url: 'get_newsletter_config_modal',
+            data: { newsletter_id: '${newsletter["id"]}' },
+            cache: false,
+            async: true,
+            complete: function (xhr, status) {
+                $('#newsletter-config-modal').html(xhr.responseText);
+            }
+        });
+    }
+
+    function saveCallback(jqXHR) {
+        if (jqXHR) {
+            var result = $.parseJSON(jqXHR.responseText);
+            var msg = result.message;
+            if (result.result == 'success') {
+                showMsg('<i class="fa fa-check"></i> ' + msg, false, true, 5000)
+            } else {
+                showMsg('<i class="fa fa-times"></i> ' + msg, false, true, 5000, true)
+            }
+        }
+
+        getNewslettersTable();
+    }
+
+    function deleteCallback() {
+        $('#newsletter-config-modal').modal('hide');
+        getNewslettersTable();
+    }
+
+    function duplicateCallback(result) {
+        // Set new newsletter id
+        $('#newsletter_id').val(result.newsletter_id);
+        // Clear friendly name
+        $('#friendly_name').val("");
+
+        saveNewsletter();
+
+        $('#newsletter-config-modal').on('hidden.bs.modal', function () {
+            loadNewsletterConfig(result.newsletter_id);
+        });
+        $('#newsletter-config-modal').modal('hide');
+    }
+
+    function saveNewsletter() {
+        // Trim all text inputs before saving
+        $('input[type=text]').val(function(_, value) {
+            return $.trim(value);
+        });
+        // Make sure simple cron value is set
+        if ($('#custom_cron').val() === '0'){
+            $("#cron_value").val(cron_widget.cron('value'));
+        }
+        doAjaxCall('set_newsletter_config', $(this), 'tabs', true, true, saveCallback);
+    }
+
+    $('#delete-newsletter-item').click(function () {
+        var msg = 'Are you sure you want to delete this <strong>${newsletter["agent_label"]}</strong> newsletter?';
+        var url = 'delete_newsletter';
+        confirmAjaxCall(url, msg, { newsletter_id: '${newsletter["id"]}' }, null, deleteCallback);
+    });
+
+    $('#duplicate-newsletter-item').click(function() {
+        var msg = 'Are you sure you want to duplicate this <strong>${newsletter["agent_label"]}</strong> newsletter?';
+        var url = 'add_newsletter_config';
+        confirmAjaxCall(url, msg, { agent_id: '${newsletter["agent_id"]}' }, null, duplicateCallback);
+    });
+
+    $('#save-newsletter-item').click(function () {
+        saveNewsletter();
+    });
+
+    $('#preview_newsletter').click(function () {
+        doAjaxCall('set_newsletter_config', $(this), 'tabs', true, false, previewNewsletter);
+    });
+
+    $('#test_newsletter').click(function () {
+        doAjaxCall('set_newsletter_config', $(this), 'tabs', true, false, sendTestNewsletter);
+    });
+
+    function previewNewsletter() {
+        showMsg('<i class="fa fa-check"></i>&nbsp; Check pop-up blocker if no response.', false, true, 2000);
+        window.open('newsletter_preview?newsletter_id=' + $('#newsletter_id').val());
+    }
+
+    function sendTestNewsletter() {
+        showMsg('<i class="fa fa-refresh fa-spin"></i>&nbsp; Sending Newsletter', false);
+        $.ajax({
+            url: 'send_newsletter',
+            data: {
+                newsletter_id: $('#newsletter_id').val(),
+                notify_action: 'test'
+            },
+            cache: false,
+            async: true,
+            success: function (data) {
+                if (data.result === 'success') {
+                    showMsg('<i class="fa fa-check"></i> ' + data.message, false, true, 5000);
+                } else {
+                    showMsg('<i class="fa fa-exclamation-circle"></i> ' + data.message, false, true, 5000, true);
+                }
+            }
+        });
+    }
+
+    $("${', '.join(['#' + c['name'] for c in newsletter['config_options'] if c.get('refresh')])}").on('change', function () {
+        // Reload modal to update certain fields
+        doAjaxCall('set_newsletter_config', $(this), 'tabs', true, false, reloadModal);
+        return false;
+    });
+
+    // Never send checkbox values directly, always substitute value in hidden input.
+    $('.checkboxes').click(function () {
+        var configToggle = $(this).data('id');
+        if ($(this).is(':checked')) {
+            $('#'+configToggle).val(1);
+        } else {
+            $('#'+configToggle).val(0);
+        }
+    });
+
+    // auto resizing textarea for custom notification message body
+    $('textarea[data-autoresize]').each(function () {
+        var offset = this.offsetHeight - this.clientHeight;
+        var resizeTextarea = function (el) {
+            $(el).css('height', 'auto').css('height', el.scrollHeight + offset);
+        };
+        $(this).on('focus keyup input', function () { resizeTextarea(this); }).removeAttr('data-autoresize');
+    });
+</script>
+% else:
+<div class="modal-dialog" role="document">
+    <div class="modal-content">
+        <div class="modal-header">
+            <button type="button" class="close" data-dismiss="modal" aria-hidden="true"><i class="fa fa-remove"></i></button>
+            <h4 class="modal-title" id="newsletter-config-modal-header">Error</h4>
+        </div>
+        <div class="modal-body" style="text-align: center">
+            <strong>
+                <i class="fa fa-exclamation-circle"></i> Failed to retrieve newsletter configuration. Check the <a href="logs">logs</a> for more info.
+            </strong>
+        </div>
+        <div class="modal-footer">
+        </div>
+    </div>
+</div>
+% endif

data/interfaces/default/newsletter_preview.html

@@ -0,0 +1,48 @@
+<%
+    import urllib
+%>
+<!doctype html>
+
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <title>Tautulli - ${title} | ${server_name}</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link href="${http_root}css/tautulli.css${cache_param}" rel="stylesheet">
+    <style>
+        * {
+            padding: 0 !important;
+            margin: 0 !important;
+        }
+    </style>
+</head>
+
+<body>
+<div id="loader" class="newsletter-loader-container">
+    <div class="newsletter-loader-message">
+        <div class="newsletter-loader"></div>
+        <br>
+        Generating Newsletter
+        <br>
+        Please wait, this may take a few minutes...
+    </div>
+</div>
+
+<script src="${http_root}js/jquery-2.1.4.min.js"></script>
+<script>
+$(document).ready(function () {
+    var frame = $('<iframe></iframe>', {
+        src: '${http_root}real_newsletter?${urllib.urlencode(kwargs) | n}',
+        frameborder: '0',
+        style: 'display: none; height: 100vh; width: 100vw;'
+    });
+    frame.on('load', function (e) {
+        $(e.target).fadeIn();
+        $('#loader').fadeOut();
+    });
+    $('body').append(frame);
+});
+</script>
+
+</body>
+</html>

data/interfaces/default/newsletters_table.html

@@ -0,0 +1,51 @@
+<%doc>
+USAGE DOCUMENTATION :: PLEASE LEAVE THIS AT THE TOP OF THIS FILE
+
+For Mako templating syntax documentation please visit: http://docs.makotemplates.org/en/latest/
+
+Filename:           newsletters_table.html
+Version:            0.1
+
+DOCUMENTATION :: END
+</%doc>
+
+<% from plexpy.newsletter_handler import NEWSLETTER_SCHED %>
+<ul class="stacked-configs list-unstyled">
+    % for newsletter in sorted(newsletters_list, key=lambda k: (k['agent_label'], k['friendly_name'], k['id'])):
+    <li class="newsletter-agent" data-id="${newsletter['id']}">
+        <span>
+            <span class="toggle-left trigger-tooltip ${'active' if newsletter['active'] else ''}" data-toggle="tooltip" data-placement="top" title="Newsletter ${'active' if newsletter['active'] else 'inactive'}"><i class="fa fa-lg fa-newspaper-o"></i></span>
+            % if newsletter['friendly_name']:
+            ${newsletter['agent_label']} &nbsp;<span class="friendly_name">(${newsletter['id']} - ${newsletter['friendly_name']})</span>
+            % else:
+            ${newsletter['agent_label']} &nbsp;<span class="friendly_name">(${newsletter['id']})</span>
+            % endif
+            <span class="toggle-right"><i class="fa fa-lg fa-cog"></i></span>
+            <span class="toggle-right friendly_name" id="newsletter-next_run-${newsletter['id']}">
+                % if NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])):
+                <% job = NEWSLETTER_SCHED.get_job('newsletter-{}'.format(newsletter['id'])) %>
+                <script>
+                    $("#newsletter-next_run-${newsletter['id']}").text(moment("${job.next_run_time}", "YYYY-MM-DD HH:mm:ssZ").fromNow())
+                </script>
+                % endif
+            </span>
+        </span>
+    </li>
+    % endfor
+    <li class="add-newsletter-agent" id="add-newsletter-agent" data-target="#add-newsletter-modal" data-toggle="modal">
+        <span>
+            <span class="toggle-left"><i class="fa fa-lg fa-newspaper-o"></i></span> Add a new newsletter agent
+            <span class="toggle-right"><i class="fa fa-lg fa-plus"></i></span>
+        </span>
+    </li>
+</ul>
+
+<script>
+    // Load newsletter config modal
+    $(".newsletter-agent").click(function () {
+        var newsletter_id = $(this).data('id');
+        loadNewsletterConfig(newsletter_id);
+    });
+
+    $('.trigger-tooltip').tooltip();
+</script>

data/interfaces/default/notifier_config.html

@@ -1,3 +1,4 @@
+% if notifier:
 <%!
     import json
     from plexpy import helpers, notifiers, users
@@ -6,9 +7,6 @@
     user_emails = [{'user': u['friendly_name'] or u['username'], 'email': u['email']} for u in users.Users().get_users() if u['email']]
     sorted(user_emails, key=lambda u: u['user'])
 %>
-% if notifier:
-<link href="${http_root}css/selectize.bootstrap3.css" rel="stylesheet" />
-<link href="${http_root}css/selectize.min.css" rel="stylesheet" />
 <div class="modal-dialog" role="document">
     <div class="modal-content">
         <div class="modal-header">
@@ -19,7 +17,7 @@
             <div class="container-fluid">
                 <div class="row">
                     <ul class="nav nav-tabs list-unstyled" role="tablist">
-                        <li role="presentation" class="active"><a href="#tabs-config" aria-controls="tabs-config" role="tab" data-toggle="tab">Configuration</a></li>
+                        <li role="presentation" class="active"><a href="#tabs-notifier_config" aria-controls="tabs-notifier_config" role="tab" data-toggle="tab">Configuration</a></li>
                         <li role="presentation"><a href="#tabs-notify_triggers" aria-controls="tabs-notify_triggers" role="tab" data-toggle="tab">Triggers</a></li>
                         <li role="presentation"><a href="#tabs-notify_conditions" aria-controls="tabs-notify_conditions" role="tab" data-toggle="tab">Conditions</a></li>
                         <li role="presentation"><a href="#tabs-notify_text" aria-controls="tabs-notify_text" role="tab" data-toggle="tab">${'Arguments' if notifier['agent_name'] == 'scripts' else 'Text'}</a></li>
@@ -28,7 +26,7 @@
                 </div>
                 <form action="set_notifier_config" method="post" class="form" id="set_notifier_config" data-parsley-validate>
                     <div class="tab-content">
-                        <div role="tabpanel" class="tab-pane active" id="tabs-config">
+                        <div role="tabpanel" class="tab-pane active" id="tabs-notifier_config">
                             <div class="row">
                                 <div class="col-md-12">
                                     <input type="hidden" id="notifier_id" name="notifier_id" value="${notifier['id']}" />
@@ -148,7 +146,7 @@
                                     % for action in available_notification_actions:
                                     <div class="checkbox">
                                         <label>
-                                            <input type="checkbox" data-id="${action['name']}" class="checkboxes" value="1" ${helpers.checked(notifier['actions'][action['name']])}> Notify on ${action['label']}
+                                            <input type="checkbox" data-id="${action['name']}" class="checkboxes" value="1" ${helpers.checked(notifier['actions'][action['name']])}> ${action['label']}
                                         </label>
                                         <p class="help-block">${action['description'] | n}</p>
                                         <input type="hidden" id="${action['name']}" name="${action['name']}" value="${notifier['actions'][action['name']]}">
@@ -164,7 +162,7 @@
                                 <a href="#notify-text-sub-modal" data-toggle="modal">Click here</a> for a description of all the parameters.
                             </p>
                             <div id="condition-widget"></div>
-                            <input type="hidden" name="custom_conditions" id="custom_conditions" />
+                            <input type="hidden" id="custom_conditions" name="custom_conditions" />
 
                             <div class="form-group">
                                 <label for="custom_conditions_logic">Condition Logic</label>
@@ -407,7 +405,7 @@
     $('#duplicate-notifier-item').click(function() {
         var msg = 'Are you sure you want to duplicate this <strong>${notifier["agent_label"]}</strong> notification agent?';
         var url = 'add_notifier_config';
-        confirmAjaxCall(url, msg, { agent_id: "${notifier['agent_id']}" }, null, duplicateCallback);
+        confirmAjaxCall(url, msg, { agent_id: '${notifier["agent_id"]}' }, null, duplicateCallback);
     });
 
     $('#save-notifier-item').click(function () {
@@ -420,7 +418,7 @@
             '<div class="form-group">' +
             '<label>Warning</label>' +
             '<p class="help-block" style="color: #eb8600;">Facebook requires HTTPS for authorization. ' +
-            'Please enable HTTPS for Tautulli under <a data-tab-destination="tabs-web_interface" data-dismiss="modal" style="cursor: pointer;">Web Interface</a>.</p>' +
+            'Please enable HTTPS for Tautulli under <a data-tab-destination="tabs-web_interface" data-dismiss="modal" data-target="#enable_https">Web Interface</a>.</p>' +
             '</div>'
         );
         $('#facebook_redirect_uri').val('HTTPS not enabled');
@@ -748,11 +746,12 @@
     });
 
     function sendTestNotification() {
+        showMsg('<i class="fa fa-refresh fa-spin"></i>&nbsp; Sending Notification', false);
         if ('${notifier["agent_name"]}' !== 'browser') {
             $.ajax({
                 url: 'send_notification',
                 data: {
-                    notifier_id: '${notifier["id"]}',
+                    notifier_id: $('#notifier_id').val(),
                     subject: $('#test_subject').val(),
                     body: $('#test_body').val(),
                     script: $('#test_script').val(),
@@ -761,13 +760,11 @@
                 },
                 cache: false,
                 async: true,
-                complete: function (xhr, status) {
-                    if (xhr.responseText.indexOf('sent') > -1) {
-                        msg = '<i class="fa fa-check"></i>&nbsp; ' + xhr.responseText;
-                        showMsg(msg, false, true, 2000);
+                success: function (data) {
+                    if (data.result === 'success') {
+                        showMsg('<i class="fa fa-check"></i> ' + data.message, false, true, 5000);
                     } else {
-                        msg = '<i class="fa fa-times"></i>&nbsp; ' + xhr.responseText;
-                        showMsg(msg, false, true, 2000, true);
+                        showMsg('<i class="fa fa-exclamation-circle"></i> ' + data.message, false, true, 5000, true);
                     }
                 }
             });

data/interfaces/default/settings.html

@@ -4,10 +4,11 @@
     import sys
 
     import plexpy
-    from plexpy import common, notifiers
+    from plexpy import common, notifiers, newsletters
     from plexpy.helpers import anon_url, checked
 
-    available_notification_agents = sorted(notifiers.available_notification_agents(), key=lambda k: k['label'])
+    available_notification_agents = sorted(notifiers.available_notification_agents(), key=lambda k: k['label'].lower())
+    available_newsletter_agents = sorted(newsletters.available_newsletter_agents(), key=lambda k: k['label'].lower())
 %>
 <%def name="headIncludes()">
 </%def>
@@ -49,8 +50,9 @@
                 <li role="presentation"><a href="#tabs-homepage" aria-controls="tabs-homepage" role="tab" data-toggle="tab">Homepage</a></li>
                 <li role="presentation"><a href="#tabs-web_interface" aria-controls="tabs-web_interface" role="tab" data-toggle="tab">Web Interface</a></li>
                 <li role="presentation"><a href="#tabs-plex_media_server" aria-controls="tabs-plex_media_server" role="tab" data-toggle="tab">Plex Media Server</a></li>
-                <li role="presentation"><a href="#tabs-notifications" aria-controls="tabs-notifications" role="tab" data-toggle="tab">Notifications</a></li>
+                <li role="presentation"><a href="#tabs-notifications" aria-controls="tabs-notifications" role="tab" data-toggle="tab">Notifications & Newsletters</a></li>
                 <li role="presentation"><a href="#tabs-notification_agents" aria-controls="tabs-notification_agents" role="tab" data-toggle="tab">Notification Agents</a></li>
+                <li role="presentation"><a href="#tabs-newsletter_agents" aria-controls="tabs-newsletter_agents" role="tab" data-toggle="tab">Newsletter Agents</a></li>
                 <li role="presentation"><a href="#tabs-import_backups" aria-controls="tabs-import_backups" role="tab" data-toggle="tab">Import & Backups</a></li>
                 <li role="presentation"><a href="#tabs-android_app" aria-controls="tabs-android_app" role="tab" data-toggle="tab">Tautulli Remote Android App <sup><small>beta</small></sup></a></li>
             </ul>
@@ -271,6 +273,17 @@
                             <h3>Activity</h3>
                         </div>
 
+                        <div class="form-group">
+                            <label for="home_refresh_interval">Activty Refresh Interval</label>
+                            <div class="row">
+                                <div class="col-md-2">
+                                    <input type="text" class="form-control" data-parsley-type="integer" id="home_refresh_interval" name="home_refresh_interval" value="${config['home_refresh_interval']}" size="5" data-parsley-min="2" data-parsley-trigger="change" data-parsley-errors-container="#home_refresh_interval_error" required>
+                                </div>
+                                <div id="home_refresh_interval_error" class="alert alert-danger settings-alert" role="alert"></div>
+                            </div>
+                            <p class="help-block">Set the interval (in seconds) to refresh the current activity on the homepage. Minimum 2.</p>
+                        </div>
+
                         <div class="padded-header">
                             <h3>Sections</h3>
                         </div>
@@ -442,6 +455,18 @@
                             </div>
                             <p class="help-block">Port to bind web server to. Note that ports below 1024 may require root.</p>
                         </div>
+                        <div class="form-group advanced-setting">
+                            <label for="http_base_url">Public Tautulli Domain</label>
+                            <div class="row">
+                                <div class="col-md-8">
+                                    <input type="text" class="form-control" id="http_base_url" name="http_base_url" value="${config['http_base_url']}" placeholder="http://mydomain.com" data-parsley-trigger="change" data-parsley-pattern="^https?:\/\/\S+$" data-parsley-errors-container="#http_base_url_error"  data-parsley-error-message="Invalid URL">
+                                </div>
+                                <div id=http_base_url_error" class="alert alert-danger settings-alert" role="alert"></div>
+                            </div>
+                            <p class="help-block">
+                                Set your public Tautulli domain for self-hosted notification images and newsletters. (e.g. http://mydomain.com)
+                            </p>
+                        </div>
                         <div class="form-group advanced-setting">
                             <label for="http_root">HTTP Root</label>
                             <div class="row">
@@ -558,7 +583,7 @@
                             <label>
                                 <input type="checkbox" name="http_hash_password" id="http_hash_password" value="1" ${config['http_hash_password']} data-parsley-trigger="change"> Hash Password in the Config File
                             </label>
-                            <span id="hashPasswordCheck" style="color: #eb8600; padding-left: 10px;"></span>
+                            <span id="hashPasswordCheck" class="settings-warning"></span>
                             <p class="help-block">Store a hashed password in the config file.<br />Warning: Your password cannot be recovered if forgotten!</p>
                         </div>
                         <input type="text" id="http_hashed_password" name="http_hashed_password" value="${config['http_hashed_password']}" style="display: none;" data-parsley-trigger="change" data-parsley-type="integer" data-parsley-range="[0, 1]"
@@ -576,14 +601,14 @@
                             <label>
                                 <input type="checkbox" class="auth-settings" name="http_plex_admin" id="http_plex_admin" value="1" ${config['http_plex_admin']} data-parsley-trigger="change"> Allow Plex Admin
                             </label>
-                            <span id="allowPlexCheck" style="color: #eb8600; padding-left: 10px;"></span>
+                            <span id="allowPlexCheck" class="settings-warning"></span>
                             <p class="help-block">Allow the Plex server admin to login as a Tautulli admin using their Plex.tv account.</p>
                         </div>
                         <div class="checkbox">
                             <label>
                                 <input type="checkbox" id="allow_guest_access" name="allow_guest_access" value="1" ${config['allow_guest_access']}> Allow Guest Access to Tautulli
                             </label>
-                            <span id="allowGuestCheck" style="color: #eb8600; padding-left: 10px;"></span>
+                            <span id="allowGuestCheck" class="settings-warning"></span>
                             <p class="help-block">Allow shared users to login to Tautulli using their Plex.tv account. Individual user access needs to be enabled from Users > Edit Mode.</p>
                         </div>
 
@@ -689,7 +714,7 @@
                             <div class="row">
                                 <div class="col-md-9">
                                     <div class="input-group">
-                                        <input type="text" class="form-control" id="pms_web_url" name="pms_web_url" value="${config['pms_web_url']}" size="30" data-parsley-trigger="change" data-parsley-pattern="^https?:\/\/\S+$|^https:\/\/app.plex.tv\/desktop$" data-parsley-errors-container="#pms_web_url_error"  data-parsley-error-message="Invalid Plex Web URL.">
+                                        <input type="text" class="form-control" id="pms_web_url" name="pms_web_url" value="${config['pms_web_url']}" size="30" data-parsley-trigger="change" data-parsley-pattern="^https?:\/\/\S+$|^https:\/\/app.plex.tv\/desktop$" data-parsley-errors-container="#pms_web_url_error"  data-parsley-error-message="Invalid Plex Web URL">
                                         <span class="input-group-btn">
                                             <button class="btn btn-form" type="button" id="test_pms_web_button">Test URL</button>
                                         </span>
@@ -765,7 +790,7 @@
                                 <input type="checkbox" id="monitor_remote_access" name="monitor_remote_access" value="1" ${config['monitor_remote_access']}> Monitor Plex Remote Access
                             </label>
                             <span id="cloudMonitorRemoteAccess" style="display: none; color: #eb8600; padding-left: 10px;"> Not available for Plex Cloud servers.</span>
-                            <span id="remoteAccessCheck" style="color: #eb8600; padding-left: 10px;"></span>
+                            <span id="remoteAccessCheck" class="settings-warning"></span>
                             <p class="help-block">Enable to have Tautulli check if remote access to the Plex Media Server goes down.</p>
                         </div>
 
@@ -911,7 +936,7 @@
                         </div>
                         <!--<div class="checkbox">
                             <label>
-                                <input type="checkbox" name="notify_recently_added_upgrade" id="notify_recently_added_upgrade" value="1" ${config['notify_recently_added_upgrade']}> Send a Notification for New Versions <span style="color: #eb8600; padding-left: 10px;">[Not working]</span>
+                                <input type="checkbox" name="notify_recently_added_upgrade" id="notify_recently_added_upgrade" value="1" ${config['notify_recently_added_upgrade']}> Send a Notification for New Versions <span class="settings-warning">[Not working]</span>
                             </label>
                             <p class="help-block">
                                 Enable to send another recently added notification when adding a new version of existing media.<br />
@@ -920,16 +945,38 @@
                         </div>-->
 
                         <div class="padded-header">
-                            <h3>3rd Party APIs</h3>
+                            <h3>Newsletters</h3>
                         </div>
 
                         <div class="checkbox">
                             <label>
-                                <input type="checkbox" name="notify_upload_posters" id="notify_upload_posters" value="1" ${config['notify_upload_posters']}> Upload Posters to Imgur for Notifications
+                                <input type="checkbox" id="newsletter_self_hosted" name="newsletter_self_hosted" value="1" ${config['newsletter_self_hosted']}> Self-Hosted Newsletters
                             </label>
-                            <p class="help-block">Enable to upload Plex posters to Imgur for notifications. Disable if posters are not being used to save bandwidth.</p>
+                            <p class="help-block">Enable to host newsletters on your own domain. This will generate a link to an HTML page where you can view the newsletter.</p>
                         </div>
-                        <div id="imgur_upload_options">
+                        <div id="self_host_newsletter_options" style="overlfow: hidden; display: ${'block' if config['newsletter_self_hosted'] == 'checked' else 'none'}">
+                            <p class="help-block" id="self_host_newsletter_message">Note: The <span class="inline-pre">${http_root}newsletter</span> endpoint on your domain must be publicly accessible from the internet.</p>
+                            <p class="help-block settings-warning base-url-warning">Warning: Public Tautulli domain not set under <a data-tab-destination="tabs-web_interface" data-target="#http_base_url">Web Interface</a>.</p>
+                        </div>
+
+                        <div class="padded-header">
+                            <h3>3rd Party APIs</h3>
+                        </div>
+
+                        <div class="form-group">
+                            <label for="notify_upload_posters">Image Hosting</label>
+                            <div class="row">
+                                <div class="col-md-6">
+                                    <select class="form-control" id="notify_upload_posters" name="notify_upload_posters">
+                                        <option value="0" ${'selected' if config['notify_upload_posters'] == 0 else ''}>Disabled</option>
+                                        <option value="1" ${'selected' if config['notify_upload_posters'] == 1 else ''}>Imgur</option>
+                                        <option value="2" ${'selected' if config['notify_upload_posters'] == 2 else ''}>Self-hosted on public Tautulli domain</option>
+                                    </select>
+                                </div>
+                            </div>
+                            <p class="help-block">Select where to host Plex images for notifications and newsletters.</p>
+                        </div>
+                        <div id="imgur_upload_options" style="overlfow: hidden; display: ${'none' if config['notify_upload_posters'] != 1 else 'block'}">
                             <div class="form-group">
                                 <label for="imgur_client_id">Imgur Client ID</label>
                                 <div class="row">
@@ -939,9 +986,14 @@
                                 </div>
                                 <p class="help-block">
                                     Enter your Imgur API client ID in order to upload posters.
-                                    You can register a new application <a href="${anon_url('https://api.imgur.com/oauth2/addclient')}" target="_blank">here</a>.<br />
+                                    You can register a new application <a href="${anon_url('https://api.imgur.com/oauth2/addclient')}" target="_blank">here</a>.
+                                </p>
                             </div>
                         </div>
+                        <div id="self_host_image_options" style="overlfow: hidden; display: ${'none' if config['notify_upload_posters'] != 2 else 'block'}">
+                            <p class="help-block" id="self_host_image_message">Note: The <span class="inline-pre">${http_root}image</span> endpoint on your domain must be publicly accessible from the internet.</p>
+                            <p class="help-block settings-warning base-url-warning">Warning: Public Tautulli domain not set under <a data-tab-destination="tabs-web_interface" data-target="#http_base_url">Web Interface</a>.</p>
+                        </div>
                         <div class="checkbox">
                             <label>
                                 <input type="checkbox" name="themoviedb_lookup" id="themoviedb_lookup" value="1" ${config['themoviedb_lookup']}> Lookup TheMovieDB Links
@@ -979,6 +1031,26 @@
 
                     </div>
 
+                    <div role="tabpanel" class="tab-pane" id="tabs-newsletter_agents">
+
+                        <div class="padded-header">
+                            <h3>Newsletter Agents</h3>
+                        </div>
+
+                        <p class="help-block">
+                            Add a new newsletter agent, or configure an existing newsletter agent by clicking the settings icon on the right.
+                        </p>
+                        <p class="help-block settings-warning" id="newsletter_upload_warning">
+                            Note: Either <a data-tab-destination="tabs-notifications" data-target="#notify_upload_posters">Image Hosting</a> on Imgur or <a data-tab-destination="tabs-notifications" data-target="#newsletter_self_hosted">Self-Hosted Newsletters</a> must be enabled.</span>
+                        </p>
+                        <br/>
+                        <div id="plexpy-newsletters-table">
+                            <div class='text-muted'><i class="fa fa-refresh fa-spin"></i> Loading newsletter agents...</div>
+                            <br>
+                        </div>
+
+                    </div>
+
                     <div role="tabpanel" class="tab-pane" id="tabs-import_backups">
 
                         <div class="padded-header">
@@ -1023,6 +1095,17 @@
                             <h3>Directories</h3>
                         </div>
 
+                        <div class="form-group">
+                            <label for="log_dir">Log Directory</label>
+                            <div class="row">
+                                <div class="col-md-6">
+                                    <input type="text" class="form-control directory-settings" id="log_dir" name="log_dir" value="${config['log_dir']}">
+                                    <div class="btn-group">
+                                        <button class="btn btn-form" type="button" id="clear_logs">Clear Logs</button>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
                         <div class="form-group">
                             <label for="backup_dir">Backup Directory</label>
                             <div class="row">
@@ -1048,13 +1131,10 @@
                             </div>
                         </div>
                         <div class="form-group">
-                            <label for="log_dir">Log Directory</label>
+                            <label for="newsletter_dir">Newsletter Directory</label>
                             <div class="row">
                                 <div class="col-md-6">
-                                    <input type="text" class="form-control directory-settings" id="log_dir" name="log_dir" value="${config['log_dir']}">
-                                    <div class="btn-group">
-                                        <button class="btn btn-form" type="button" id="clear_logs">Clear Logs</button>
-                                    </div>
+                                    <input type="text" class="form-control directory-settings" id="newsletter_dir" name="newsletter_dir" value="${config['newsletter_dir']}">
                                 </div>
                             </div>
                         </div>
@@ -1081,7 +1161,7 @@
                         <p class="form-group">
                             <label>Registered Devices</label>
                             <p class="help-block">Register a new device using a QR code, or configure an existing device by clicking the settings icon on the right.</p>
-                            <p id="app_api_msg" style="color: #eb8600;">The API must be enabled under <a data-tab-destination="tabs-web_interface" style="cursor: pointer;">Web Interface</a> to use the app.</p>
+                            <p id="app_api_msg" style="color: #eb8600;">The API must be enabled under <a data-tab-destination="tabs-web_interface" data-target="#api_enabled">Web Interface</a> to use the app.</p>
                             <div class="row">
                                 <div id="plexpy-mobile-devices-table" class="col-md-12">
                                     <div class='text-muted'><i class="fa fa-refresh fa-spin"></i> Loading registered devices...</div>
@@ -1248,7 +1328,36 @@
         </div>
     </div>
 </div>
+<div id="add-newsletter-modal" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="add-newsletter-modal">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-hidden="true"><i class="fa fa-remove"></i></button>
+                <h4 class="modal-title">Add a Newsletter Agent</h4>
+            </div>
+            <div class="modal-body">
+                <div class="container-fluid">
+                    <div class="row">
+                        <div class="col-md-12">
+                            <ul class="stacked-configs list-unstyled">
+                                % for agent in available_newsletter_agents:
+                                <li class="new-newsletter-agent" data-id="${agent['id']}">
+                                    <span>${agent['label']}</span>
+                                </li>
+                                % endfor
+                            </ul>
+                        </div>
+                    </div>
+                </div>
+            </div>
+            <div class="modal-footer">
+                <input type="button" class="btn btn-bright" data-dismiss="modal" value="Cancel">
+            </div>
+        </div>
+    </div>
+</div>
 <div id="notifier-config-modal" class="modal fade wide" tabindex="-1" role="dialog" aria-labelledby="notifier-config-modal"></div>
+<div id="newsletter-config-modal" class="modal fade wide" tabindex="-1" role="dialog" aria-labelledby="newsletter-config-modal"></div>
 <div id="notify-text-sub-modal" class="modal fade wide" tabindex="-1" role="dialog" aria-labelledby="notify-text-sub-modal">
     <div class="modal-dialog" role="document">
         <div class="modal-content">
@@ -1404,6 +1513,53 @@
 </div>
 <div id="notifier-text-preview-modal" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="notifier-text-preview-modal">
 </div>
+<div id="newsletter-text-sub-modal" class="modal fade wide" tabindex="-1" role="dialog" aria-labelledby="newsletter-text-sub-modal">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal" aria-hidden="true">
+                    <i class="fa fa-remove"></i>
+                </button>
+                <h4 class="modal-title">Newsletter Parameters</h4>
+            </div>
+            <div class="modal-body">
+                <div>
+                    <p class="help-block">
+                        If the value for a selected parameter cannot be provided, it will display as blank.
+                    </p>
+                    % for category in common.NEWSLETTER_PARAMETERS:
+                    <table class="notification-params">
+                        <thead>
+                            <tr>
+                                <th colspan="2">
+                                    ${category['category']}
+                                </th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            % for parameter in category['parameters']:
+                            <tr>
+                                <td><strong>{${parameter['value']}}</strong></td>
+                                <td>
+                                    ${parameter['description']}
+                                    % if parameter.get('example'):
+                                    <span class="small-muted">(${parameter['example']})</span>
+                                    % endif
+                                    % if parameter.get('help_text'):
+                                    <p class="small-muted">(${parameter['help_text']})</p>
+                                    % endif
+                                </td>
+                            </tr>
+                            % endfor
+                        </tbody>
+                    </table>
+                    % endfor
+                </div>
+            </div>
+            <div class="modal-footer"></div>
+        </div>
+    </div>
+</div>
 <div id="changelog-modal" class="modal fade wide" tabindex="-1" role="dialog" aria-labelledby="changelog-modal">
     <div class="modal-dialog" role="document">
         <div class="modal-content">
@@ -1538,6 +1694,29 @@
         });
     }
 
+    function getNewslettersTable() {
+        $.ajax({
+            url: 'get_newsletters_table',
+            cache: false,
+            async: true,
+            complete: function(xhr, status) {
+                $("#plexpy-newsletters-table").html(xhr.responseText);
+            }
+        });
+    }
+
+    function loadNewsletterConfig(newsletter_id) {
+        $.ajax({
+            url: 'get_newsletter_config_modal',
+            data: { newsletter_id: newsletter_id },
+            cache: false,
+            async: true,
+            complete: function (xhr, status) {
+                $("#newsletter-config-modal").html(xhr.responseText).modal('show');
+            }
+        });
+    }
+
     function getMobileDevicesTable() {
         $.ajax({
             url: 'get_mobile_devices_table',
@@ -1610,6 +1789,7 @@ $(document).ready(function() {
         getConfigurationTable();
         getSchedulerTable();
         getNotifiersTable();
+        getNewslettersTable();
         getMobileDevicesTable();
         loadUpdateDistros();
         settingsChanged = false;
@@ -1646,8 +1826,8 @@ $(document).ready(function() {
     initConfigCheckbox('#enable_https');
     initConfigCheckbox('#https_create_cert');
     initConfigCheckbox('#check_github');
-    initConfigCheckbox('#notify_upload_posters');
     initConfigCheckbox('#monitor_pms_updates');
+    initConfigCheckbox('#newsletter_self_hosted');
 
     $('#menu_link_shutdown').click(function() {
         $('#confirm-message').text("Are you sure you want to shutdown Tautulli?");
@@ -1693,6 +1873,7 @@ $(document).ready(function() {
     getConfigurationTable();
     getSchedulerTable();
     getNotifiersTable();
+    getNewslettersTable();
     getMobileDevicesTable();
 
     $('#changelog-modal-link').on('click', function (e) {
@@ -2310,6 +2491,32 @@ $(document).ready(function() {
         });
     });
 
+    // Add a new newsletter agent
+    $('.new-newsletter-agent').click(function () {
+        $.ajax({
+            url: 'add_newsletter_config',
+            data: { agent_id: $(this).data('id') },
+            cache: false,
+            async: true,
+            complete: function (xhr, status) {
+                var result = $.parseJSON(xhr.responseText);
+                var msg = result.message;
+                $('#add-newsletter-modal').modal('hide');
+                if (result.result == 'success') {
+                    showMsg('<i class="fa fa-check"></i> ' + msg, false, true, 5000);
+                    loadNewsletterConfig(result.newsletter_id);
+                } else {
+                    showMsg('<i class="fa fa-times"></i> ' + msg, false, true, 5000, true);
+                }
+                getNewslettersTable();
+            }
+        });
+    });
+
+    $('#http_base_url').change(function () {
+       $(this).val($(this).val().replace(/\/*$/, ''));
+    });
+
     function apiEnabled() {
         var api_enabled = $('#api_enabled').prop('checked');
         $('#app_api_msg').toggle(!(api_enabled));
@@ -2319,9 +2526,62 @@ $(document).ready(function() {
         apiEnabled();
     });
 
+    function imageUpload() {
+        var upload_val = $('#notify_upload_posters').val();
+        if (upload_val === '1') {
+            $('#imgur_upload_options').slideDown();
+        } else {
+            $('#imgur_upload_options').slideUp();
+        }
+        if (upload_val === '2') {
+            $('#self_host_image_options').slideDown();
+        } else {
+            $('#self_host_image_options').slideUp();
+        }
+    }
+    $('#notify_upload_posters').change(function () {
+        imageUpload();
+    });
+
+    function baseURLSet() {
+        if ($('#http_base_url').val()) {
+            $('.base-url-warning').hide();
+        } else {
+            $('.base-url-warning').show();
+        }
+    }
+    baseURLSet();
+
+    $('#http_base_url').change(function () {
+       baseURLSet();
+    });
+
+    function newsletterUploadEnabled() {
+        if ($('#notify_upload_posters').val() === '1' || $('#newsletter_self_hosted').is(':checked')) {
+            $('#newsletter_upload_warning').hide();
+        } else {
+            $('#newsletter_upload_warning').show();
+        }
+    }
+    newsletterUploadEnabled();
+
+    $('#notify_upload_posters, #newsletter_self_hosted').change(function () {
+        baseURLSet();
+        newsletterUploadEnabled();
+    });
+
     $('body').on('click', 'a[data-tab-destination]', function () {
         var tab = $(this).data('tab-destination');
         $("a[href=#" + tab + "]").click();
+        var scroll_destination = $(this).data('target');
+        if (scroll_destination) {
+            if ($(scroll_destination).closest('.advanced-setting').length && !$('#menu_link_show_advanced_settings').hasClass('active')) {
+                $('#menu_link_show_advanced_settings').click()
+            }
+            var body_container = $('.body-container')
+            var scroll_pos = scroll_destination ? body_container.scrollTop() + $(scroll_destination).offset().top - 100 : 0;
+            body_container.animate({scrollTop: scroll_pos});
+        }
     });
 });
 </script>

lib/apscheduler/__init__.py

@@ -1,5 +1,10 @@
-version_info = (3, 0, 1)
-version = '3.0.1'
-release = '3.0.1'
+from pkg_resources import get_distribution, DistributionNotFound
 
-__version__ = release  # PEP 396
+try:
+    release = get_distribution('APScheduler').version.split('-')[0]
+except DistributionNotFound:
+    release = '3.5.0'
+
+version_info = tuple(int(x) if x.isdigit() else x for x in release.split('.'))
+version = __version__ = '.'.join(str(x) for x in version_info[:3])
+del get_distribution, DistributionNotFound

lib/apscheduler/events.py

@@ -1,25 +1,33 @@
-__all__ = ('EVENT_SCHEDULER_START', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_EXECUTOR_ADDED', 'EVENT_EXECUTOR_REMOVED',
-           'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED', 'EVENT_JOB_ADDED',
-           'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED', 'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED',
+__all__ = ('EVENT_SCHEDULER_STARTED', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_SCHEDULER_PAUSED',
+           'EVENT_SCHEDULER_RESUMED', 'EVENT_EXECUTOR_ADDED', 'EVENT_EXECUTOR_REMOVED',
+           'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED',
+           'EVENT_JOB_ADDED', 'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED',
+           'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED', 'EVENT_JOB_SUBMITTED', 'EVENT_JOB_MAX_INSTANCES',
            'SchedulerEvent', 'JobEvent', 'JobExecutionEvent')
 
 
-EVENT_SCHEDULER_START = 1
-EVENT_SCHEDULER_SHUTDOWN = 2
-EVENT_EXECUTOR_ADDED = 4
-EVENT_EXECUTOR_REMOVED = 8
-EVENT_JOBSTORE_ADDED = 16
-EVENT_JOBSTORE_REMOVED = 32
-EVENT_ALL_JOBS_REMOVED = 64
-EVENT_JOB_ADDED = 128
-EVENT_JOB_REMOVED = 256
-EVENT_JOB_MODIFIED = 512
-EVENT_JOB_EXECUTED = 1024
-EVENT_JOB_ERROR = 2048
-EVENT_JOB_MISSED = 4096
-EVENT_ALL = (EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN | EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED |
+EVENT_SCHEDULER_STARTED = EVENT_SCHEDULER_START = 2 ** 0
+EVENT_SCHEDULER_SHUTDOWN = 2 ** 1
+EVENT_SCHEDULER_PAUSED = 2 ** 2
+EVENT_SCHEDULER_RESUMED = 2 ** 3
+EVENT_EXECUTOR_ADDED = 2 ** 4
+EVENT_EXECUTOR_REMOVED = 2 ** 5
+EVENT_JOBSTORE_ADDED = 2 ** 6
+EVENT_JOBSTORE_REMOVED = 2 ** 7
+EVENT_ALL_JOBS_REMOVED = 2 ** 8
+EVENT_JOB_ADDED = 2 ** 9
+EVENT_JOB_REMOVED = 2 ** 10
+EVENT_JOB_MODIFIED = 2 ** 11
+EVENT_JOB_EXECUTED = 2 ** 12
+EVENT_JOB_ERROR = 2 ** 13
+EVENT_JOB_MISSED = 2 ** 14
+EVENT_JOB_SUBMITTED = 2 ** 15
+EVENT_JOB_MAX_INSTANCES = 2 ** 16
+EVENT_ALL = (EVENT_SCHEDULER_STARTED | EVENT_SCHEDULER_SHUTDOWN | EVENT_SCHEDULER_PAUSED |
+             EVENT_SCHEDULER_RESUMED | EVENT_EXECUTOR_ADDED | EVENT_EXECUTOR_REMOVED |
+             EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED | EVENT_ALL_JOBS_REMOVED |
              EVENT_JOB_ADDED | EVENT_JOB_REMOVED | EVENT_JOB_MODIFIED | EVENT_JOB_EXECUTED |
-             EVENT_JOB_ERROR | EVENT_JOB_MISSED)
+             EVENT_JOB_ERROR | EVENT_JOB_MISSED | EVENT_JOB_SUBMITTED | EVENT_JOB_MAX_INSTANCES)
 
 
 class SchedulerEvent(object):
@@ -55,9 +63,21 @@ class JobEvent(SchedulerEvent):
         self.jobstore = jobstore
 
 
+class JobSubmissionEvent(JobEvent):
+    """
+    An event that concerns the submission of a job to its executor.
+
+    :ivar scheduled_run_times: a list of datetimes when the job was intended to run
+    """
+
+    def __init__(self, code, job_id, jobstore, scheduled_run_times):
+        super(JobSubmissionEvent, self).__init__(code, job_id, jobstore)
+        self.scheduled_run_times = scheduled_run_times
+
+
 class JobExecutionEvent(JobEvent):
     """
-    An event that concerns the execution of individual jobs.
+    An event that concerns the running of a job within its executor.
 
     :ivar scheduled_run_time: the time when the job was scheduled to be run
     :ivar retval: the return value of the successfully executed job
@@ -65,7 +85,8 @@ class JobExecutionEvent(JobEvent):
     :ivar traceback: a formatted traceback for the exception
     """
 
-    def __init__(self, code, job_id, jobstore, scheduled_run_time, retval=None, exception=None, traceback=None):
+    def __init__(self, code, job_id, jobstore, scheduled_run_time, retval=None, exception=None,
+                 traceback=None):
         super(JobExecutionEvent, self).__init__(code, job_id, jobstore)
         self.scheduled_run_time = scheduled_run_time
         self.retval = retval

lib/apscheduler/executors/asyncio.py

@@ -1,28 +1,60 @@
 from __future__ import absolute_import
+
 import sys
 
 from apscheduler.executors.base import BaseExecutor, run_job
 
+try:
+    from asyncio import iscoroutinefunction
+    from apscheduler.executors.base_py3 import run_coroutine_job
+except ImportError:
+    from trollius import iscoroutinefunction
+    run_coroutine_job = None
+
 
 class AsyncIOExecutor(BaseExecutor):
     """
     Runs jobs in the default executor of the event loop.
 
+    If the job function is a native coroutine function, it is scheduled to be run directly in the
+    event loop as soon as possible. All other functions are run in the event loop's default
+    executor which is usually a thread pool.
+
     Plugin alias: ``asyncio``
     """
 
     def start(self, scheduler, alias):
         super(AsyncIOExecutor, self).start(scheduler, alias)
         self._eventloop = scheduler._eventloop
+        self._pending_futures = set()
+
+    def shutdown(self, wait=True):
+        # There is no way to honor wait=True without converting this method into a coroutine method
+        for f in self._pending_futures:
+            if not f.done():
+                f.cancel()
+
+        self._pending_futures.clear()
 
     def _do_submit_job(self, job, run_times):
         def callback(f):
+            self._pending_futures.discard(f)
             try:
                 events = f.result()
-            except:
+            except BaseException:
                 self._run_job_error(job.id, *sys.exc_info()[1:])
             else:
                 self._run_job_success(job.id, events)
 
-        f = self._eventloop.run_in_executor(None, run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        if iscoroutinefunction(job.func):
+            if run_coroutine_job is not None:
+                coro = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
+                f = self._eventloop.create_task(coro)
+            else:
+                raise Exception('Executing coroutine based jobs is not supported with Trollius')
+        else:
+            f = self._eventloop.run_in_executor(None, run_job, job, job._jobstore_alias, run_times,
+                                                self._logger.name)
+
         f.add_done_callback(callback)
+        self._pending_futures.add(f)

lib/apscheduler/executors/base.py

@@ -8,13 +8,15 @@ import sys
 from pytz import utc
 import six
 
-from apscheduler.events import JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED
+from apscheduler.events import (
+    JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED)
 
 
 class MaxInstancesReachedError(Exception):
     def __init__(self, job):
         super(MaxInstancesReachedError, self).__init__(
-            'Job "%s" has already reached its maximum number of instances (%d)' % (job.id, job.max_instances))
+            'Job "%s" has already reached its maximum number of instances (%d)' %
+            (job.id, job.max_instances))
 
 
 class BaseExecutor(six.with_metaclass(ABCMeta, object)):
@@ -30,13 +32,14 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
 
     def start(self, scheduler, alias):
         """
-        Called by the scheduler when the scheduler is being started or when the executor is being added to an already
-        running scheduler.
+        Called by the scheduler when the scheduler is being started or when the executor is being
+        added to an already running scheduler.
 
-        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this executor
+        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting
+            this executor
         :param str|unicode alias: alias of this executor as it was assigned to the scheduler
-        """
 
+        """
         self._scheduler = scheduler
         self._lock = scheduler._create_lock()
         self._logger = logging.getLogger('apscheduler.executors.%s' % alias)
@@ -45,7 +48,8 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         """
         Shuts down this executor.
 
-        :param bool wait: ``True`` to wait until all submitted jobs have been executed
+        :param bool wait: ``True`` to wait until all submitted jobs
+            have been executed
         """
 
     def submit_job(self, job, run_times):
@@ -53,10 +57,12 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         Submits job for execution.
 
         :param Job job: job to execute
-        :param list[datetime] run_times: list of datetimes specifying when the job should have been run
-        :raises MaxInstancesReachedError: if the maximum number of allowed instances for this job has been reached
-        """
+        :param list[datetime] run_times: list of datetimes specifying
+            when the job should have been run
+        :raises MaxInstancesReachedError: if the maximum number of
+            allowed instances for this job has been reached
 
+        """
         assert self._lock is not None, 'This executor has not been started yet'
         with self._lock:
             if self._instances[job.id] >= job.max_instances:
@@ -70,50 +76,71 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         """Performs the actual task of scheduling `run_job` to be called."""
 
     def _run_job_success(self, job_id, events):
-        """Called by the executor with the list of generated events when `run_job` has been successfully called."""
+        """
+        Called by the executor with the list of generated events when :func:`run_job` has been
+        successfully called.
 
+        """
         with self._lock:
             self._instances[job_id] -= 1
+            if self._instances[job_id] == 0:
+                del self._instances[job_id]
 
         for event in events:
             self._scheduler._dispatch_event(event)
 
     def _run_job_error(self, job_id, exc, traceback=None):
         """Called by the executor with the exception if there is an error  calling `run_job`."""
-
         with self._lock:
             self._instances[job_id] -= 1
+            if self._instances[job_id] == 0:
+                del self._instances[job_id]
 
         exc_info = (exc.__class__, exc, traceback)
         self._logger.error('Error running job %s', job_id, exc_info=exc_info)
 
 
 def run_job(job, jobstore_alias, run_times, logger_name):
-    """Called by executors to run the job. Returns a list of scheduler events to be dispatched by the scheduler."""
+    """
+    Called by executors to run the job. Returns a list of scheduler events to be dispatched by the
+    scheduler.
 
+    """
     events = []
     logger = logging.getLogger(logger_name)
     for run_time in run_times:
-        # See if the job missed its run time window, and handle possible misfires accordingly
+        # See if the job missed its run time window, and handle
+        # possible misfires accordingly
         if job.misfire_grace_time is not None:
             difference = datetime.now(utc) - run_time
             grace_time = timedelta(seconds=job.misfire_grace_time)
             if difference > grace_time:
-                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias, run_time))
+                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias,
+                                                run_time))
                 logger.warning('Run time of job "%s" was missed by %s', job, difference)
                 continue
 
         logger.info('Running job "%s" (scheduled at %s)', job, run_time)
         try:
             retval = job.func(*job.args, **job.kwargs)
-        except:
+        except BaseException:
             exc, tb = sys.exc_info()[1:]
             formatted_tb = ''.join(format_tb(tb))
-            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time, exception=exc,
-                                            traceback=formatted_tb))
+            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time,
+                                            exception=exc, traceback=formatted_tb))
             logger.exception('Job "%s" raised an exception', job)
+
+            # This is to prevent cyclic references that would lead to memory leaks
+            if six.PY2:
+                sys.exc_clear()
+                del tb
             else:
-            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time, retval=retval))
+                import traceback
+                traceback.clear_frames(tb)
+                del tb
+        else:
+            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time,
+                                            retval=retval))
             logger.info('Job "%s" executed successfully', job)
 
     return events

lib/apscheduler/executors/base_py3.py

@@ -0,0 +1,41 @@
+import logging
+import sys
+from datetime import datetime, timedelta
+from traceback import format_tb
+
+from pytz import utc
+
+from apscheduler.events import (
+    JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED)
+
+
+async def run_coroutine_job(job, jobstore_alias, run_times, logger_name):
+    """Coroutine version of run_job()."""
+    events = []
+    logger = logging.getLogger(logger_name)
+    for run_time in run_times:
+        # See if the job missed its run time window, and handle possible misfires accordingly
+        if job.misfire_grace_time is not None:
+            difference = datetime.now(utc) - run_time
+            grace_time = timedelta(seconds=job.misfire_grace_time)
+            if difference > grace_time:
+                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias,
+                                                run_time))
+                logger.warning('Run time of job "%s" was missed by %s', job, difference)
+                continue
+
+        logger.info('Running job "%s" (scheduled at %s)', job, run_time)
+        try:
+            retval = await job.func(*job.args, **job.kwargs)
+        except BaseException:
+            exc, tb = sys.exc_info()[1:]
+            formatted_tb = ''.join(format_tb(tb))
+            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time,
+                                            exception=exc, traceback=formatted_tb))
+            logger.exception('Job "%s" raised an exception', job)
+        else:
+            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time,
+                                            retval=retval))
+            logger.info('Job "%s" executed successfully', job)
+
+    return events

lib/apscheduler/executors/debug.py

@@ -5,7 +5,8 @@ from apscheduler.executors.base import BaseExecutor, run_job
 
 class DebugExecutor(BaseExecutor):
     """
-    A special executor that executes the target callable directly instead of deferring it to a thread or process.
+    A special executor that executes the target callable directly instead of deferring it to a
+    thread or process.
 
     Plugin alias: ``debug``
     """
@@ -13,7 +14,7 @@ class DebugExecutor(BaseExecutor):
     def _do_submit_job(self, job, run_times):
         try:
             events = run_job(job, job._jobstore_alias, run_times, self._logger.name)
-        except:
+        except BaseException:
             self._run_job_error(job.id, *sys.exc_info()[1:])
         else:
             self._run_job_success(job.id, events)

lib/apscheduler/executors/gevent.py

@@ -21,9 +21,10 @@ class GeventExecutor(BaseExecutor):
         def callback(greenlet):
             try:
                 events = greenlet.get()
-            except:
+            except BaseException:
                 self._run_job_error(job.id, *sys.exc_info()[1:])
             else:
                 self._run_job_success(job.id, events)
 
-        gevent.spawn(run_job, job, job._jobstore_alias, run_times, self._logger.name).link(callback)
+        gevent.spawn(run_job, job, job._jobstore_alias, run_times, self._logger.name).\
+            link(callback)

lib/apscheduler/executors/tornado.py

@@ -0,0 +1,54 @@
+from __future__ import absolute_import
+
+import sys
+from concurrent.futures import ThreadPoolExecutor
+
+from tornado.gen import convert_yielded
+
+from apscheduler.executors.base import BaseExecutor, run_job
+
+try:
+    from inspect import iscoroutinefunction
+    from apscheduler.executors.base_py3 import run_coroutine_job
+except ImportError:
+    def iscoroutinefunction(func):
+        return False
+
+
+class TornadoExecutor(BaseExecutor):
+    """
+    Runs jobs either in a thread pool or directly on the I/O loop.
+
+    If the job function is a native coroutine function, it is scheduled to be run directly in the
+    I/O loop as soon as possible. All other functions are run in a thread pool.
+
+    Plugin alias: ``tornado``
+
+    :param int max_workers: maximum number of worker threads in the thread pool
+    """
+
+    def __init__(self, max_workers=10):
+        super(TornadoExecutor, self).__init__()
+        self.executor = ThreadPoolExecutor(max_workers)
+
+    def start(self, scheduler, alias):
+        super(TornadoExecutor, self).start(scheduler, alias)
+        self._ioloop = scheduler._ioloop
+
+    def _do_submit_job(self, job, run_times):
+        def callback(f):
+            try:
+                events = f.result()
+            except BaseException:
+                self._run_job_error(job.id, *sys.exc_info()[1:])
+            else:
+                self._run_job_success(job.id, events)
+
+        if iscoroutinefunction(job.func):
+            f = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
+        else:
+            f = self.executor.submit(run_job, job, job._jobstore_alias, run_times,
+                                     self._logger.name)
+
+        f = convert_yielded(f)
+        f.add_done_callback(callback)

lib/apscheduler/executors/twisted.py

@@ -21,5 +21,5 @@ class TwistedExecutor(BaseExecutor):
             else:
                 self._run_job_error(job.id, result.value, result.tb)
 
-        self._reactor.getThreadPool().callInThreadWithCallback(callback, run_job, job, job._jobstore_alias, run_times,
-                                                               self._logger.name)
+        self._reactor.getThreadPool().callInThreadWithCallback(
+            callback, run_job, job, job._jobstore_alias, run_times, self._logger.name)

lib/apscheduler/job.py

@@ -4,8 +4,9 @@ from uuid import uuid4
 import six
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args, \
-    convert_to_datetime
+from apscheduler.util import (
+    ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args,
+    convert_to_datetime)
 
 
 class Job(object):
@@ -21,13 +22,20 @@ class Job(object):
     :var bool coalesce: whether to only run the job once when several run times are due
     :var trigger: the trigger object that controls the schedule of this job
     :var str executor: the name of the executor that will run this job
-    :var int misfire_grace_time: the time (in seconds) how much this job's execution is allowed to be late
-    :var int max_instances: the maximum number of concurrently executing instances allowed for this job
+    :var int misfire_grace_time: the time (in seconds) how much this job's execution is allowed to
+        be late
+    :var int max_instances: the maximum number of concurrently executing instances allowed for this
+        job
     :var datetime.datetime next_run_time: the next scheduled run time of this job
+
+    .. note::
+        The ``misfire_grace_time`` has some non-obvious effects on job execution. See the
+        :ref:`missed-job-executions` section in the documentation for an in-depth explanation.
     """
 
-    __slots__ = ('_scheduler', '_jobstore_alias', 'id', 'trigger', 'executor', 'func', 'func_ref', 'args', 'kwargs',
-                 'name', 'misfire_grace_time', 'coalesce', 'max_instances', 'next_run_time')
+    __slots__ = ('_scheduler', '_jobstore_alias', 'id', 'trigger', 'executor', 'func', 'func_ref',
+                 'args', 'kwargs', 'name', 'misfire_grace_time', 'coalesce', 'max_instances',
+                 'next_run_time')
 
     def __init__(self, scheduler, id=None, **kwargs):
         super(Job, self).__init__()
@@ -38,53 +46,69 @@ class Job(object):
     def modify(self, **changes):
         """
         Makes the given changes to this job and saves it in the associated job store.
+
         Accepted keyword arguments are the same as the variables on this class.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.modify_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.modify_job(self.id, self._jobstore_alias, **changes)
+        return self
 
     def reschedule(self, trigger, **trigger_args):
         """
         Shortcut for switching the trigger on this job.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.reschedule_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.reschedule_job(self.id, self._jobstore_alias, trigger, **trigger_args)
+        return self
 
     def pause(self):
         """
         Temporarily suspend the execution of this job.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.pause_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.pause_job(self.id, self._jobstore_alias)
+        return self
 
     def resume(self):
         """
         Resume the schedule of this job if previously paused.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.resume_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.resume_job(self.id, self._jobstore_alias)
+        return self
 
     def remove(self):
         """
         Unschedules this job and removes it from its associated job store.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.remove_job`
-        """
 
+        """
         self._scheduler.remove_job(self.id, self._jobstore_alias)
 
     @property
     def pending(self):
-        """Returns ``True`` if the referenced job is still waiting to be added to its designated job store."""
+        """
+        Returns ``True`` if the referenced job is still waiting to be added to its designated job
+        store.
 
+        """
         return self._jobstore_alias is None
 
     #
@@ -97,8 +121,8 @@ class Job(object):
 
         :type now: datetime.datetime
         :rtype: list[datetime.datetime]
-        """
 
+        """
         run_times = []
         next_run_time = self.next_run_time
         while next_run_time and next_run_time <= now:
@@ -108,8 +132,11 @@ class Job(object):
         return run_times
 
     def _modify(self, **changes):
-        """Validates the changes to the Job and makes the modifications if and only if all of them validate."""
+        """
+        Validates the changes to the Job and makes the modifications if and only if all of them
+        validate.
 
+        """
         approved = {}
 
         if 'id' in changes:
@@ -125,7 +152,7 @@ class Job(object):
             args = changes.pop('args') if 'args' in changes else self.args
             kwargs = changes.pop('kwargs') if 'kwargs' in changes else self.kwargs
 
-            if isinstance(func, str):
+            if isinstance(func, six.string_types):
                 func_ref = func
                 func = ref_to_obj(func)
             elif callable(func):
@@ -177,7 +204,8 @@ class Job(object):
         if 'trigger' in changes:
             trigger = changes.pop('trigger')
             if not isinstance(trigger, BaseTrigger):
-                raise TypeError('Expected a trigger instance, got %s instead' % trigger.__class__.__name__)
+                raise TypeError('Expected a trigger instance, got %s instead' %
+                                trigger.__class__.__name__)
 
             approved['trigger'] = trigger
 
@@ -189,10 +217,12 @@ class Job(object):
 
         if 'next_run_time' in changes:
             value = changes.pop('next_run_time')
-            approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone, 'next_run_time')
+            approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone,
+                                                            'next_run_time')
 
         if changes:
-            raise AttributeError('The following are not modifiable attributes of Job: %s' % ', '.join(changes))
+            raise AttributeError('The following are not modifiable attributes of Job: %s' %
+                                 ', '.join(changes))
 
         for key, value in six.iteritems(approved):
             setattr(self, key, value)
@@ -200,9 +230,10 @@ class Job(object):
     def __getstate__(self):
         # Don't allow this Job to be serialized if the function reference could not be determined
         if not self.func_ref:
-            raise ValueError('This Job cannot be serialized since the reference to its callable (%r) could not be '
-                             'determined. Consider giving a textual reference (module:function name) instead.' %
-                             (self.func,))
+            raise ValueError(
+                'This Job cannot be serialized since the reference to its callable (%r) could not '
+                'be determined. Consider giving a textual reference (module:function name) '
+                'instead.' % (self.func,))
 
         return {
             'version': 1,
@@ -221,7 +252,8 @@ class Job(object):
 
     def __setstate__(self, state):
         if state.get('version', 1) > 1:
-            raise ValueError('Job has version %s, but only version 1 can be handled' % state['version'])
+            raise ValueError('Job has version %s, but only version 1 can be handled' %
+                             state['version'])
 
         self.id = state['id']
         self.func_ref = state['func']
@@ -245,8 +277,13 @@ class Job(object):
         return '<Job (id=%s name=%s)>' % (repr_escape(self.id), repr_escape(self.name))
 
     def __str__(self):
-        return '%s (trigger: %s, next run at: %s)' % (repr_escape(self.name), repr_escape(str(self.trigger)),
-                                                      datetime_repr(self.next_run_time))
+        return repr_escape(self.__unicode__())
 
     def __unicode__(self):
-        return six.u('%s (trigger: %s, next run at: %s)') % (self.name, self.trigger, datetime_repr(self.next_run_time))
+        if hasattr(self, 'next_run_time'):
+            status = ('next run at: ' + datetime_repr(self.next_run_time) if
+                      self.next_run_time else 'paused')
+        else:
+            status = 'pending'
+
+        return u'%s (trigger: %s, %s)' % (self.name, self.trigger, status)

lib/apscheduler/jobstores/base.py

@@ -8,23 +8,27 @@ class JobLookupError(KeyError):
     """Raised when the job store cannot find a job for update or removal."""
 
     def __init__(self, job_id):
-        super(JobLookupError, self).__init__(six.u('No job by the id of %s was found') % job_id)
+        super(JobLookupError, self).__init__(u'No job by the id of %s was found' % job_id)
 
 
 class ConflictingIdError(KeyError):
     """Raised when the uniqueness of job IDs is being violated."""
 
     def __init__(self, job_id):
-        super(ConflictingIdError, self).__init__(six.u('Job identifier (%s) conflicts with an existing job') % job_id)
+        super(ConflictingIdError, self).__init__(
+            u'Job identifier (%s) conflicts with an existing job' % job_id)
 
 
 class TransientJobError(ValueError):
-    """Raised when an attempt to add transient (with no func_ref) job to a persistent job store is detected."""
+    """
+    Raised when an attempt to add transient (with no func_ref) job to a persistent job store is
+    detected.
+    """
 
     def __init__(self, job_id):
         super(TransientJobError, self).__init__(
-            six.u('Job (%s) cannot be added to this job store because a reference to the callable could not be '
-                  'determined.') % job_id)
+            u'Job (%s) cannot be added to this job store because a reference to the callable '
+            u'could not be determined.' % job_id)
 
 
 class BaseJobStore(six.with_metaclass(ABCMeta)):
@@ -36,10 +40,11 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
 
     def start(self, scheduler, alias):
         """
-        Called by the scheduler when the scheduler is being started or when the job store is being added to an already
-        running scheduler.
+        Called by the scheduler when the scheduler is being started or when the job store is being
+        added to an already running scheduler.
 
-        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this job store
+        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting
+            this job store
         :param str|unicode alias: alias of this job store as it was assigned to the scheduler
         """
 
@@ -50,13 +55,22 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     def shutdown(self):
         """Frees any resources still bound to this job store."""
 
+    def _fix_paused_jobs_sorting(self, jobs):
+        for i, job in enumerate(jobs):
+            if job.next_run_time is not None:
+                if i > 0:
+                    paused_jobs = jobs[:i]
+                    del jobs[:i]
+                    jobs.extend(paused_jobs)
+                break
+
     @abstractmethod
     def lookup_job(self, job_id):
         """
         Returns a specific job, or ``None`` if it isn't found..
 
-        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned job to
-        point to the scheduler and itself, respectively.
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of
+        the returned job to point to the scheduler and itself, respectively.
 
         :param str|unicode job_id: identifier of the job
         :rtype: Job
@@ -75,7 +89,8 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     @abstractmethod
     def get_next_run_time(self):
         """
-        Returns the earliest run time of all the jobs stored in this job store, or ``None`` if there are no active jobs.
+        Returns the earliest run time of all the jobs stored in this job store, or ``None`` if
+        there are no active jobs.
 
         :rtype: datetime.datetime
         """
@@ -83,11 +98,12 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     @abstractmethod
     def get_all_jobs(self):
         """
-        Returns a list of all jobs in this job store. The returned jobs should be sorted by next run time (ascending).
-        Paused jobs (next_run_time is None) should be sorted last.
+        Returns a list of all jobs in this job store.
+        The returned jobs should be sorted by next run time (ascending).
+        Paused jobs (next_run_time == None) should be sorted last.
 
-        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned jobs to
-        point to the scheduler and itself, respectively.
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of
+        the returned jobs to point to the scheduler and itself, respectively.
 
         :rtype: list[Job]
         """

lib/apscheduler/jobstores/memory.py

@@ -13,7 +13,8 @@ class MemoryJobStore(BaseJobStore):
 
     def __init__(self):
         super(MemoryJobStore, self).__init__()
-        self._jobs = []  # list of (job, timestamp), sorted by next_run_time and job id (ascending)
+        # list of (job, timestamp), sorted by next_run_time and job id (ascending)
+        self._jobs = []
         self._jobs_index = {}  # id -> (job, timestamp) lookup table
 
     def lookup_job(self, job_id):
@@ -80,13 +81,13 @@ class MemoryJobStore(BaseJobStore):
 
     def _get_job_index(self, timestamp, job_id):
         """
-        Returns the index of the given job, or if it's not found, the index where the job should be inserted based on
-        the given timestamp.
+        Returns the index of the given job, or if it's not found, the index where the job should be
+        inserted based on the given timestamp.
 
         :type timestamp: int
         :type job_id: str
-        """
 
+        """
         lo, hi = 0, len(self._jobs)
         timestamp = float('inf') if timestamp is None else timestamp
         while lo < hi:

lib/apscheduler/jobstores/mongodb.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import warnings
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
 from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
@@ -19,16 +20,18 @@ except ImportError:  # pragma: nocover
 
 class MongoDBJobStore(BaseJobStore):
     """
-    Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to pymongo's `MongoClient
+    Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to
+    pymongo's `MongoClient
     <http://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`_.
 
     Plugin alias: ``mongodb``
 
     :param str database: database to store jobs in
     :param str collection: collection to store jobs in
-    :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of providing connection
-                   arguments
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of
+        providing connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
     """
 
     def __init__(self, database='apscheduler', collection='jobs', client=None,
@@ -42,14 +45,23 @@ class MongoDBJobStore(BaseJobStore):
             raise ValueError('The "collection" parameter must not be empty')
 
         if client:
-            self.connection = maybe_ref(client)
+            self.client = maybe_ref(client)
         else:
             connect_args.setdefault('w', 1)
-            self.connection = MongoClient(**connect_args)
+            self.client = MongoClient(**connect_args)
 
-        self.collection = self.connection[database][collection]
+        self.collection = self.client[database][collection]
+
+    def start(self, scheduler, alias):
+        super(MongoDBJobStore, self).start(scheduler, alias)
         self.collection.ensure_index('next_run_time', sparse=True)
 
+    @property
+    def connection(self):
+        warnings.warn('The "connection" member is deprecated -- use "client" instead',
+                      DeprecationWarning)
+        return self.client
+
     def lookup_job(self, job_id):
         document = self.collection.find_one(job_id, ['job_state'])
         return self._reconstitute_job(document['job_state']) if document else None
@@ -59,12 +71,15 @@ class MongoDBJobStore(BaseJobStore):
         return self._get_jobs({'next_run_time': {'$lte': timestamp}})
 
     def get_next_run_time(self):
-        document = self.collection.find_one({'next_run_time': {'$ne': None}}, fields=['next_run_time'],
+        document = self.collection.find_one({'next_run_time': {'$ne': None}},
+                                            projection=['next_run_time'],
                                             sort=[('next_run_time', ASCENDING)])
         return utc_timestamp_to_datetime(document['next_run_time']) if document else None
 
     def get_all_jobs(self):
-        return self._get_jobs({})
+        jobs = self._get_jobs({})
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
 
     def add_job(self, job):
         try:
@@ -83,7 +98,7 @@ class MongoDBJobStore(BaseJobStore):
         }
         result = self.collection.update({'_id': job.id}, {'$set': changes})
         if result and result['n'] == 0:
-            raise JobLookupError(id)
+            raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
         result = self.collection.remove(job_id)
@@ -94,7 +109,7 @@ class MongoDBJobStore(BaseJobStore):
         self.collection.remove()
 
     def shutdown(self):
-        self.connection.disconnect()
+        self.client.close()
 
     def _reconstitute_job(self, job_state):
         job_state = pickle.loads(job_state)
@@ -107,11 +122,13 @@ class MongoDBJobStore(BaseJobStore):
     def _get_jobs(self, conditions):
         jobs = []
         failed_job_ids = []
-        for document in self.collection.find(conditions, ['_id', 'job_state'], sort=[('next_run_time', ASCENDING)]):
+        for document in self.collection.find(conditions, ['_id', 'job_state'],
+                                             sort=[('next_run_time', ASCENDING)]):
             try:
                 jobs.append(self._reconstitute_job(document['job_state']))
-            except:
-                self._logger.exception('Unable to restore job "%s" -- removing it', document['_id'])
+            except BaseException:
+                self._logger.exception('Unable to restore job "%s" -- removing it',
+                                       document['_id'])
                 failed_job_ids.append(document['_id'])
 
         # Remove all the jobs we failed to restore
@@ -121,4 +138,4 @@ class MongoDBJobStore(BaseJobStore):
         return jobs
 
     def __repr__(self):
-        return '<%s (client=%s)>' % (self.__class__.__name__, self.connection)
+        return '<%s (client=%s)>' % (self.__class__.__name__, self.client)

lib/apscheduler/jobstores/redis.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
+from datetime import datetime
 
+from pytz import utc
 import six
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
@@ -19,14 +21,16 @@ except ImportError:  # pragma: nocover
 
 class RedisJobStore(BaseJobStore):
     """
-    Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's StrictRedis.
+    Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's
+    :class:`~redis.StrictRedis`.
 
     Plugin alias: ``redis``
 
     :param int db: the database number to store jobs in
     :param str jobs_key: key to store jobs in
     :param str run_times_key: key to store the jobs' run times in
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
     """
 
     def __init__(self, db=0, jobs_key='apscheduler.jobs', run_times_key='apscheduler.run_times',
@@ -65,7 +69,8 @@ class RedisJobStore(BaseJobStore):
     def get_all_jobs(self):
         job_states = self.redis.hgetall(self.jobs_key)
         jobs = self._reconstitute_jobs(six.iteritems(job_states))
-        return sorted(jobs, key=lambda job: job.next_run_time)
+        paused_sort_key = datetime(9999, 12, 31, tzinfo=utc)
+        return sorted(jobs, key=lambda job: job.next_run_time or paused_sort_key)
 
     def add_job(self, job):
         if self.redis.hexists(self.jobs_key, job.id):
@@ -73,7 +78,9 @@ class RedisJobStore(BaseJobStore):
 
         with self.redis.pipeline() as pipe:
             pipe.multi()
-            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
+                                                          self.pickle_protocol))
+            if job.next_run_time:
                 pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
             pipe.execute()
 
@@ -82,7 +89,8 @@ class RedisJobStore(BaseJobStore):
             raise JobLookupError(job.id)
 
         with self.redis.pipeline() as pipe:
-            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
+                                                          self.pickle_protocol))
             if job.next_run_time:
                 pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
             else:
@@ -121,7 +129,7 @@ class RedisJobStore(BaseJobStore):
         for job_id, job_state in job_states:
             try:
                 jobs.append(self._reconstitute_job(job_state))
-            except:
+            except BaseException:
                 self._logger.exception('Unable to restore job "%s" -- removing it', job_id)
                 failed_job_ids.append(job_id)
 

lib/apscheduler/jobstores/rethinkdb.py

@@ -0,0 +1,153 @@
+from __future__ import absolute_import
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    import rethinkdb as r
+except ImportError:  # pragma: nocover
+    raise ImportError('RethinkDBJobStore requires rethinkdb installed')
+
+
+class RethinkDBJobStore(BaseJobStore):
+    """
+    Stores jobs in a RethinkDB database. Any leftover keyword arguments are directly passed to
+    rethinkdb's `RethinkdbClient <http://www.rethinkdb.com/api/#connect>`_.
+
+    Plugin alias: ``rethinkdb``
+
+    :param str database: database to store jobs in
+    :param str collection: collection to store jobs in
+    :param client: a :class:`rethinkdb.net.Connection` instance to use instead of providing
+        connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    """
+
+    def __init__(self, database='apscheduler', table='jobs', client=None,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(RethinkDBJobStore, self).__init__()
+
+        if not database:
+            raise ValueError('The "database" parameter must not be empty')
+        if not table:
+            raise ValueError('The "table" parameter must not be empty')
+
+        self.database = database
+        self.table = table
+        self.client = client
+        self.pickle_protocol = pickle_protocol
+        self.connect_args = connect_args
+        self.conn = None
+
+    def start(self, scheduler, alias):
+        super(RethinkDBJobStore, self).start(scheduler, alias)
+
+        if self.client:
+            self.conn = maybe_ref(self.client)
+        else:
+            self.conn = r.connect(db=self.database, **self.connect_args)
+
+        if self.database not in r.db_list().run(self.conn):
+            r.db_create(self.database).run(self.conn)
+
+        if self.table not in r.table_list().run(self.conn):
+            r.table_create(self.table).run(self.conn)
+
+        if 'next_run_time' not in r.table(self.table).index_list().run(self.conn):
+            r.table(self.table).index_create('next_run_time').run(self.conn)
+
+        self.table = r.db(self.database).table(self.table)
+
+    def lookup_job(self, job_id):
+        results = list(self.table.get_all(job_id).pluck('job_state').run(self.conn))
+        return self._reconstitute_job(results[0]['job_state']) if results else None
+
+    def get_due_jobs(self, now):
+        return self._get_jobs(r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
+
+    def get_next_run_time(self):
+        results = list(
+            self.table
+            .filter(r.row['next_run_time'] != None)  # flake8: noqa
+            .order_by(r.asc('next_run_time'))
+            .map(lambda x: x['next_run_time'])
+            .limit(1)
+            .run(self.conn)
+        )
+        return utc_timestamp_to_datetime(results[0]) if results else None
+
+    def get_all_jobs(self):
+        jobs = self._get_jobs()
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
+
+    def add_job(self, job):
+        job_dict = {
+            'id': job.id,
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+        }
+        results = self.table.insert(job_dict).run(self.conn)
+        if results['errors'] > 0:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        changes = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+        }
+        results = self.table.get_all(job.id).update(changes).run(self.conn)
+        skipped = False in map(lambda x: results[x] == 0, results.keys())
+        if results['skipped'] > 0 or results['errors'] > 0 or not skipped:
+            raise JobLookupError(job.id)
+
+    def remove_job(self, job_id):
+        results = self.table.get_all(job_id).delete().run(self.conn)
+        if results['deleted'] + results['skipped'] != 1:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        self.table.delete().run(self.conn)
+
+    def shutdown(self):
+        self.conn.close()
+
+    def _reconstitute_job(self, job_state):
+        job_state = pickle.loads(job_state)
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self, predicate=None):
+        jobs = []
+        failed_job_ids = []
+        query = (self.table.filter(r.row['next_run_time'] != None).filter(predicate) if
+                 predicate else self.table)
+        query = query.order_by('next_run_time', 'id').pluck('id', 'job_state')
+
+        for document in query.run(self.conn):
+            try:
+                jobs.append(self._reconstitute_job(document['job_state']))
+            except:
+                self._logger.exception('Unable to restore job "%s" -- removing it', document['id'])
+                failed_job_ids.append(document['id'])
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            r.expr(failed_job_ids).for_each(
+                lambda job_id: self.table.get_all(job_id).delete()).run(self.conn)
+
+        return jobs
+
+    def __repr__(self):
+        connection = self.conn
+        return '<%s (connection=%s)>' % (self.__class__.__name__, connection)

lib/apscheduler/jobstores/sqlalchemy.py

@@ -10,29 +10,38 @@ except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from sqlalchemy import create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select
+    from sqlalchemy import (
+        create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select)
     from sqlalchemy.exc import IntegrityError
+    from sqlalchemy.sql.expression import null
 except ImportError:  # pragma: nocover
     raise ImportError('SQLAlchemyJobStore requires SQLAlchemy installed')
 
 
 class SQLAlchemyJobStore(BaseJobStore):
     """
-    Stores jobs in a database table using SQLAlchemy. The table will be created if it doesn't exist in the database.
+    Stores jobs in a database table using SQLAlchemy.
+    The table will be created if it doesn't exist in the database.
 
     Plugin alias: ``sqlalchemy``
 
-    :param str url: connection string (see `SQLAlchemy documentation
-                    <http://docs.sqlalchemy.org/en/latest/core/engines.html?highlight=create_engine#database-urls>`_
-                    on this)
-    :param engine: an SQLAlchemy Engine to use instead of creating a new one based on ``url``
+    :param str url: connection string (see
+        :ref:`SQLAlchemy documentation <sqlalchemy:database_urls>` on this)
+    :param engine: an SQLAlchemy :class:`~sqlalchemy.engine.Engine` to use instead of creating a
+        new one based on ``url``
     :param str tablename: name of the table to store jobs in
-    :param metadata: a :class:`~sqlalchemy.MetaData` instance to use instead of creating a new one
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param metadata: a :class:`~sqlalchemy.schema.MetaData` instance to use instead of creating a
+        new one
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    :param str tableschema: name of the (existing) schema in the target database where the table
+        should be
+    :param dict engine_options: keyword arguments to :func:`~sqlalchemy.create_engine`
+        (ignored if ``engine`` is given)
     """
 
     def __init__(self, url=None, engine=None, tablename='apscheduler_jobs', metadata=None,
-                 pickle_protocol=pickle.HIGHEST_PROTOCOL):
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, tableschema=None, engine_options=None):
         super(SQLAlchemyJobStore, self).__init__()
         self.pickle_protocol = pickle_protocol
         metadata = maybe_ref(metadata) or MetaData()
@@ -40,18 +49,22 @@ class SQLAlchemyJobStore(BaseJobStore):
         if engine:
             self.engine = maybe_ref(engine)
         elif url:
-            self.engine = create_engine(url)
+            self.engine = create_engine(url, **(engine_options or {}))
         else:
             raise ValueError('Need either "engine" or "url" defined')
 
-        # 191 = max key length in MySQL for InnoDB/utf8mb4 tables, 25 = precision that translates to an 8-byte float
+        # 191 = max key length in MySQL for InnoDB/utf8mb4 tables,
+        # 25 = precision that translates to an 8-byte float
         self.jobs_t = Table(
             tablename, metadata,
             Column('id', Unicode(191, _warn_on_bytestring=False), primary_key=True),
             Column('next_run_time', Float(25), index=True),
-            Column('job_state', LargeBinary, nullable=False)
+            Column('job_state', LargeBinary, nullable=False),
+            schema=tableschema
         )
 
+    def start(self, scheduler, alias):
+        super(SQLAlchemyJobStore, self).start(scheduler, alias)
         self.jobs_t.create(self.engine, True)
 
     def lookup_job(self, job_id):
@@ -64,13 +77,16 @@ class SQLAlchemyJobStore(BaseJobStore):
         return self._get_jobs(self.jobs_t.c.next_run_time <= timestamp)
 
     def get_next_run_time(self):
-        selectable = select([self.jobs_t.c.next_run_time]).where(self.jobs_t.c.next_run_time != None).\
+        selectable = select([self.jobs_t.c.next_run_time]).\
+            where(self.jobs_t.c.next_run_time != null()).\
             order_by(self.jobs_t.c.next_run_time).limit(1)
         next_run_time = self.engine.execute(selectable).scalar()
         return utc_timestamp_to_datetime(next_run_time)
 
     def get_all_jobs(self):
-        return self._get_jobs()
+        jobs = self._get_jobs()
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
 
     def add_job(self, job):
         insert = self.jobs_t.insert().values(**{
@@ -116,13 +132,14 @@ class SQLAlchemyJobStore(BaseJobStore):
 
     def _get_jobs(self, *conditions):
         jobs = []
-        selectable = select([self.jobs_t.c.id, self.jobs_t.c.job_state]).order_by(self.jobs_t.c.next_run_time)
+        selectable = select([self.jobs_t.c.id, self.jobs_t.c.job_state]).\
+            order_by(self.jobs_t.c.next_run_time)
         selectable = selectable.where(*conditions) if conditions else selectable
         failed_job_ids = set()
         for row in self.engine.execute(selectable):
             try:
                 jobs.append(self._reconstitute_job(row.job_state))
-            except:
+            except BaseException:
                 self._logger.exception('Unable to restore job "%s" -- removing it', row.id)
                 failed_job_ids.add(row.id)
 

lib/apscheduler/jobstores/zookeeper.py

@@ -0,0 +1,179 @@
+from __future__ import absolute_import
+
+import os
+from datetime import datetime
+
+from pytz import utc
+from kazoo.exceptions import NoNodeError, NodeExistsError
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from kazoo.client import KazooClient
+except ImportError:  # pragma: nocover
+    raise ImportError('ZooKeeperJobStore requires Kazoo installed')
+
+
+class ZooKeeperJobStore(BaseJobStore):
+    """
+    Stores jobs in a ZooKeeper tree. Any leftover keyword arguments are directly passed to
+    kazoo's `KazooClient
+    <http://kazoo.readthedocs.io/en/latest/api/client.html>`_.
+
+    Plugin alias: ``zookeeper``
+
+    :param str path: path to store jobs in
+    :param client: a :class:`~kazoo.client.KazooClient` instance to use instead of
+        providing connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    """
+
+    def __init__(self, path='/apscheduler', client=None, close_connection_on_exit=False,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(ZooKeeperJobStore, self).__init__()
+        self.pickle_protocol = pickle_protocol
+        self.close_connection_on_exit = close_connection_on_exit
+
+        if not path:
+            raise ValueError('The "path" parameter must not be empty')
+
+        self.path = path
+
+        if client:
+            self.client = maybe_ref(client)
+        else:
+            self.client = KazooClient(**connect_args)
+        self._ensured_path = False
+
+    def _ensure_paths(self):
+        if not self._ensured_path:
+            self.client.ensure_path(self.path)
+        self._ensured_path = True
+
+    def start(self, scheduler, alias):
+        super(ZooKeeperJobStore, self).start(scheduler, alias)
+        if not self.client.connected:
+            self.client.start()
+
+    def lookup_job(self, job_id):
+        self._ensure_paths()
+        node_path = os.path.join(self.path, job_id)
+        try:
+            content, _ = self.client.get(node_path)
+            doc = pickle.loads(content)
+            job = self._reconstitute_job(doc['job_state'])
+            return job
+        except BaseException:
+            return None
+
+    def get_due_jobs(self, now):
+        timestamp = datetime_to_utc_timestamp(now)
+        jobs = [job_def['job'] for job_def in self._get_jobs()
+                if job_def['next_run_time'] is not None and job_def['next_run_time'] <= timestamp]
+        return jobs
+
+    def get_next_run_time(self):
+        next_runs = [job_def['next_run_time'] for job_def in self._get_jobs()
+                     if job_def['next_run_time'] is not None]
+        return utc_timestamp_to_datetime(min(next_runs)) if len(next_runs) > 0 else None
+
+    def get_all_jobs(self):
+        jobs = [job_def['job'] for job_def in self._get_jobs()]
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
+
+    def add_job(self, job):
+        self._ensure_paths()
+        node_path = os.path.join(self.path,  str(job.id))
+        value = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': job.__getstate__()
+        }
+        data = pickle.dumps(value, self.pickle_protocol)
+        try:
+            self.client.create(node_path, value=data)
+        except NodeExistsError:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        self._ensure_paths()
+        node_path = os.path.join(self.path,  str(job.id))
+        changes = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': job.__getstate__()
+        }
+        data = pickle.dumps(changes, self.pickle_protocol)
+        try:
+            self.client.set(node_path, value=data)
+        except NoNodeError:
+            raise JobLookupError(job.id)
+
+    def remove_job(self, job_id):
+        self._ensure_paths()
+        node_path = os.path.join(self.path,  str(job_id))
+        try:
+            self.client.delete(node_path)
+        except NoNodeError:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        try:
+            self.client.delete(self.path, recursive=True)
+        except NoNodeError:
+            pass
+        self._ensured_path = False
+
+    def shutdown(self):
+        if self.close_connection_on_exit:
+            self.client.stop()
+            self.client.close()
+
+    def _reconstitute_job(self, job_state):
+        job_state = job_state
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self):
+        self._ensure_paths()
+        jobs = []
+        failed_job_ids = []
+        all_ids = self.client.get_children(self.path)
+        for node_name in all_ids:
+            try:
+                node_path = os.path.join(self.path, node_name)
+                content, _ = self.client.get(node_path)
+                doc = pickle.loads(content)
+                job_def = {
+                    'job_id': node_name,
+                    'next_run_time': doc['next_run_time'] if doc['next_run_time'] else None,
+                    'job_state': doc['job_state'],
+                    'job': self._reconstitute_job(doc['job_state']),
+                    'creation_time': _.ctime
+                }
+                jobs.append(job_def)
+            except BaseException:
+                self._logger.exception('Unable to restore job "%s" -- removing it' % node_name)
+                failed_job_ids.append(node_name)
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            for failed_id in failed_job_ids:
+                self.remove_job(failed_id)
+        paused_sort_key = datetime(9999, 12, 31, tzinfo=utc)
+        return sorted(jobs, key=lambda job_def: (job_def['job'].next_run_time or paused_sort_key,
+                                                 job_def['creation_time']))
+
+    def __repr__(self):
+        self._logger.exception('<%s (client=%s)>' % (self.__class__.__name__, self.client))
+        return '<%s (client=%s)>' % (self.__class__.__name__, self.client)

lib/apscheduler/schedulers/asyncio.py

@@ -1,5 +1,5 @@
 from __future__ import absolute_import
-from functools import wraps
+from functools import wraps, partial
 
 from apscheduler.schedulers.base import BaseScheduler
 from apscheduler.util import maybe_ref
@@ -10,13 +10,15 @@ except ImportError:  # pragma: nocover
     try:
         import trollius as asyncio
     except ImportError:
-        raise ImportError('AsyncIOScheduler requires either Python 3.4 or the asyncio package installed')
+        raise ImportError(
+            'AsyncIOScheduler requires either Python 3.4 or the asyncio package installed')
 
 
 def run_in_event_loop(func):
     @wraps(func)
     def wrapper(self, *args, **kwargs):
-        self._eventloop.call_soon_threadsafe(func, self, *args, **kwargs)
+        wrapped = partial(func, self, *args, **kwargs)
+        self._eventloop.call_soon_threadsafe(wrapped)
     return wrapper
 
 
@@ -24,6 +26,8 @@ class AsyncIOScheduler(BaseScheduler):
     """
     A scheduler that runs on an asyncio (:pep:`3156`) event loop.
 
+    The default executor can run jobs based on native coroutines (``async def``).
+
     Extra options:
 
     ============== =============================================================
@@ -34,10 +38,6 @@ class AsyncIOScheduler(BaseScheduler):
     _eventloop = None
     _timeout = None
 
-    def start(self):
-        super(AsyncIOScheduler, self).start()
-        self.wakeup()
-
     @run_in_event_loop
     def shutdown(self, wait=True):
         super(AsyncIOScheduler, self).shutdown(wait)

lib/apscheduler/schedulers/background.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+
 from threading import Thread, Event
 
 from apscheduler.schedulers.base import BaseScheduler
@@ -13,11 +14,12 @@ class BackgroundScheduler(BlockingScheduler):
 
     Extra options:
 
-    ========== ============================================================================================
-    ``daemon`` Set the ``daemon`` option in the background thread (defaults to ``True``,
-               see `the documentation <https://docs.python.org/3.4/library/threading.html#thread-objects>`_
+    ========== =============================================================================
+    ``daemon`` Set the ``daemon`` option in the background thread (defaults to ``True``, see
+               `the documentation
+               <https://docs.python.org/3.4/library/threading.html#thread-objects>`_
                for further details)
-    ========== ============================================================================================
+    ========== =============================================================================
     """
 
     _thread = None
@@ -26,14 +28,14 @@ class BackgroundScheduler(BlockingScheduler):
         self._daemon = asbool(config.pop('daemon', True))
         super(BackgroundScheduler, self)._configure(config)
 
-    def start(self):
-        BaseScheduler.start(self)
+    def start(self, *args, **kwargs):
         self._event = Event()
+        BaseScheduler.start(self, *args, **kwargs)
         self._thread = Thread(target=self._main_loop, name='APScheduler')
         self._thread.daemon = self._daemon
         self._thread.start()
 
-    def shutdown(self, wait=True):
-        super(BackgroundScheduler, self).shutdown(wait)
+    def shutdown(self, *args, **kwargs):
+        super(BackgroundScheduler, self).shutdown(*args, **kwargs)
         self._thread.join()
         del self._thread

lib/apscheduler/schedulers/base.py

@@ -1,9 +1,11 @@
 from __future__ import print_function
+
 from abc import ABCMeta, abstractmethod
 from collections import MutableMapping
 from threading import RLock
-from datetime import datetime
+from datetime import datetime, timedelta
 from logging import getLogger
+import warnings
 import sys
 
 from pkg_resources import iter_entry_points
@@ -19,20 +21,39 @@ from apscheduler.job import Job
 from apscheduler.triggers.base import BaseTrigger
 from apscheduler.util import asbool, asint, astimezone, maybe_ref, timedelta_seconds, undefined
 from apscheduler.events import (
-    SchedulerEvent, JobEvent, EVENT_SCHEDULER_START, EVENT_SCHEDULER_SHUTDOWN, EVENT_JOBSTORE_ADDED,
-    EVENT_JOBSTORE_REMOVED, EVENT_ALL, EVENT_JOB_MODIFIED, EVENT_JOB_REMOVED, EVENT_JOB_ADDED, EVENT_EXECUTOR_ADDED,
-    EVENT_EXECUTOR_REMOVED, EVENT_ALL_JOBS_REMOVED)
+    SchedulerEvent, JobEvent, JobSubmissionEvent, EVENT_SCHEDULER_START, EVENT_SCHEDULER_SHUTDOWN,
+    EVENT_JOBSTORE_ADDED, EVENT_JOBSTORE_REMOVED, EVENT_ALL, EVENT_JOB_MODIFIED, EVENT_JOB_REMOVED,
+    EVENT_JOB_ADDED, EVENT_EXECUTOR_ADDED, EVENT_EXECUTOR_REMOVED, EVENT_ALL_JOBS_REMOVED,
+    EVENT_JOB_SUBMITTED, EVENT_JOB_MAX_INSTANCES, EVENT_SCHEDULER_RESUMED, EVENT_SCHEDULER_PAUSED)
+
+#: constant indicating a scheduler's stopped state
+STATE_STOPPED = 0
+#: constant indicating a scheduler's running state (started and processing jobs)
+STATE_RUNNING = 1
+#: constant indicating a scheduler's paused state (started but not processing jobs)
+STATE_PAUSED = 2
 
 
 class BaseScheduler(six.with_metaclass(ABCMeta)):
     """
-    Abstract base class for all schedulers. Takes the following keyword arguments:
+    Abstract base class for all schedulers.
 
-    :param str|logging.Logger logger: logger to use for the scheduler's logging (defaults to apscheduler.scheduler)
+    Takes the following keyword arguments:
+
+    :param str|logging.Logger logger: logger to use for the scheduler's logging (defaults to
+        apscheduler.scheduler)
     :param str|datetime.tzinfo timezone: the default time zone (defaults to the local timezone)
+    :param int|float jobstore_retry_interval: the minimum number of seconds to wait between
+        retries in the scheduler's main loop if the job store raises an exception when getting
+        the list of due jobs
     :param dict job_defaults: default values for newly added jobs
-    :param dict jobstores: a dictionary of job store alias -> job store instance or configuration dict
-    :param dict executors: a dictionary of executor alias -> executor instance or configuration dict
+    :param dict jobstores: a dictionary of job store alias -> job store instance or configuration
+        dict
+    :param dict executors: a dictionary of executor alias -> executor instance or configuration
+        dict
+
+    :ivar int state: current running state of the scheduler (one of the following constants from
+        ``apscheduler.schedulers.base``: ``STATE_STOPPED``, ``STATE_RUNNING``, ``STATE_PAUSED``)
 
     .. seealso:: :ref:`scheduler-config`
     """
@@ -43,7 +64,6 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
     _executor_classes = {}
     _jobstore_plugins = dict((ep.name, ep) for ep in iter_entry_points('apscheduler.jobstores'))
     _jobstore_classes = {}
-    _stopped = True
 
     #
     # Public API
@@ -58,28 +78,34 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         self._listeners = []
         self._listeners_lock = self._create_lock()
         self._pending_jobs = []
+        self.state = STATE_STOPPED
         self.configure(gconfig, **options)
 
     def configure(self, gconfig={}, prefix='apscheduler.', **options):
         """
-        Reconfigures the scheduler with the given options. Can only be done when the scheduler isn't running.
+        Reconfigures the scheduler with the given options.
 
-        :param dict gconfig: a "global" configuration dictionary whose values can be overridden by keyword arguments to
-                             this method
-        :param str|unicode prefix: pick only those keys from ``gconfig`` that are prefixed with this string
-                                   (pass an empty string or ``None`` to use all keys)
+        Can only be done when the scheduler isn't running.
+
+        :param dict gconfig: a "global" configuration dictionary whose values can be overridden by
+            keyword arguments to this method
+        :param str|unicode prefix: pick only those keys from ``gconfig`` that are prefixed with
+            this string (pass an empty string or ``None`` to use all keys)
         :raises SchedulerAlreadyRunningError: if the scheduler is already running
-        """
 
-        if self.running:
+        """
+        if self.state != STATE_STOPPED:
             raise SchedulerAlreadyRunningError
 
-        # If a non-empty prefix was given, strip it from the keys in the global configuration dict
+        # If a non-empty prefix was given, strip it from the keys in the
+        # global configuration dict
         if prefix:
             prefixlen = len(prefix)
-            gconfig = dict((key[prefixlen:], value) for key, value in six.iteritems(gconfig) if key.startswith(prefix))
+            gconfig = dict((key[prefixlen:], value) for key, value in six.iteritems(gconfig)
+                           if key.startswith(prefix))
 
-        # Create a structure from the dotted options (e.g. "a.b.c = d" -> {'a': {'b': {'c': 'd'}}})
+        # Create a structure from the dotted options
+        # (e.g. "a.b.c = d" -> {'a': {'b': {'c': 'd'}}})
         config = {}
         for key, value in six.iteritems(gconfig):
             parts = key.split('.')
@@ -94,15 +120,15 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         config.update(options)
         self._configure(config)
 
-    @abstractmethod
-    def start(self):
+    def start(self, paused=False):
         """
-        Starts the scheduler. The details of this process depend on the implementation.
+        Start the configured executors and job stores and begin processing scheduled jobs.
 
+        :param bool paused: if ``True``, don't start job processing until :meth:`resume` is called
         :raises SchedulerAlreadyRunningError: if the scheduler is already running
-        """
 
-        if self.running:
+        """
+        if self.state != STATE_STOPPED:
             raise SchedulerAlreadyRunningError
 
         with self._executors_lock:
@@ -125,29 +151,33 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
             # Schedule all pending jobs
             for job, jobstore_alias, replace_existing in self._pending_jobs:
-                self._real_add_job(job, jobstore_alias, replace_existing, False)
+                self._real_add_job(job, jobstore_alias, replace_existing)
             del self._pending_jobs[:]
 
-        self._stopped = False
+        self.state = STATE_PAUSED if paused else STATE_RUNNING
         self._logger.info('Scheduler started')
-
-        # Notify listeners that the scheduler has been started
         self._dispatch_event(SchedulerEvent(EVENT_SCHEDULER_START))
 
+        if not paused:
+            self.wakeup()
+
     @abstractmethod
     def shutdown(self, wait=True):
         """
-        Shuts down the scheduler. Does not interrupt any currently running jobs.
+        Shuts down the scheduler, along with its executors and job stores.
+
+        Does not interrupt any currently running jobs.
 
         :param bool wait: ``True`` to wait until all currently executing jobs have finished
         :raises SchedulerNotRunningError: if the scheduler has not been started yet
-        """
 
-        if not self.running:
+        """
+        if self.state == STATE_STOPPED:
             raise SchedulerNotRunningError
 
-        self._stopped = True
+        self.state = STATE_STOPPED
 
+        with self._jobstores_lock, self._executors_lock:
             # Shut down all executors
             for executor in six.itervalues(self._executors):
                 executor.shutdown(wait)
@@ -159,36 +189,71 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         self._logger.info('Scheduler has been shut down')
         self._dispatch_event(SchedulerEvent(EVENT_SCHEDULER_SHUTDOWN))
 
+    def pause(self):
+        """
+        Pause job processing in the scheduler.
+
+        This will prevent the scheduler from waking up to do job processing until :meth:`resume`
+        is called. It will not however stop any already running job processing.
+
+        """
+        if self.state == STATE_STOPPED:
+            raise SchedulerNotRunningError
+        elif self.state == STATE_RUNNING:
+            self.state = STATE_PAUSED
+            self._logger.info('Paused scheduler job processing')
+            self._dispatch_event(SchedulerEvent(EVENT_SCHEDULER_PAUSED))
+
+    def resume(self):
+        """Resume job processing in the scheduler."""
+        if self.state == STATE_STOPPED:
+            raise SchedulerNotRunningError
+        elif self.state == STATE_PAUSED:
+            self.state = STATE_RUNNING
+            self._logger.info('Resumed scheduler job processing')
+            self._dispatch_event(SchedulerEvent(EVENT_SCHEDULER_RESUMED))
+            self.wakeup()
+
     @property
     def running(self):
-        return not self._stopped
+        """
+        Return ``True`` if the scheduler has been started.
+
+        This is a shortcut for ``scheduler.state != STATE_STOPPED``.
+
+        """
+        return self.state != STATE_STOPPED
 
     def add_executor(self, executor, alias='default', **executor_opts):
         """
-        Adds an executor to this scheduler. Any extra keyword arguments will be passed to the executor plugin's
-        constructor, assuming that the first argument is the name of an executor plugin.
+        Adds an executor to this scheduler.
 
-        :param str|unicode|apscheduler.executors.base.BaseExecutor executor: either an executor instance or the name of
-            an executor plugin
+        Any extra keyword arguments will be passed to the executor plugin's constructor, assuming
+        that the first argument is the name of an executor plugin.
+
+        :param str|unicode|apscheduler.executors.base.BaseExecutor executor: either an executor
+            instance or the name of an executor plugin
         :param str|unicode alias: alias for the scheduler
         :raises ValueError: if there is already an executor by the given alias
-        """
 
+        """
         with self._executors_lock:
             if alias in self._executors:
-                raise ValueError('This scheduler already has an executor by the alias of "%s"' % alias)
+                raise ValueError('This scheduler already has an executor by the alias of "%s"' %
+                                 alias)
 
             if isinstance(executor, BaseExecutor):
                 self._executors[alias] = executor
             elif isinstance(executor, six.string_types):
-                self._executors[alias] = executor = self._create_plugin_instance('executor', executor, executor_opts)
+                self._executors[alias] = executor = self._create_plugin_instance(
+                    'executor', executor, executor_opts)
             else:
                 raise TypeError('Expected an executor instance or a string, got %s instead' %
                                 executor.__class__.__name__)
 
             # Start the executor right away if the scheduler is running
-            if self.running:
-                executor.start(self)
+            if self.state != STATE_STOPPED:
+                executor.start(self, alias)
 
         self._dispatch_event(SchedulerEvent(EVENT_EXECUTOR_ADDED, alias))
 
@@ -197,10 +262,11 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         Removes the executor by the given alias from this scheduler.
 
         :param str|unicode alias: alias of the executor
-        :param bool shutdown: ``True`` to shut down the executor after removing it
-        """
+        :param bool shutdown: ``True`` to shut down the executor after
+            removing it
 
-        with self._jobstores_lock:
+        """
+        with self._executors_lock:
             executor = self._lookup_executor(alias)
             del self._executors[alias]
 
@@ -211,35 +277,39 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
     def add_jobstore(self, jobstore, alias='default', **jobstore_opts):
         """
-        Adds a job store to this scheduler. Any extra keyword arguments will be passed to the job store plugin's
-        constructor, assuming that the first argument is the name of a job store plugin.
+        Adds a job store to this scheduler.
+
+        Any extra keyword arguments will be passed to the job store plugin's constructor, assuming
+        that the first argument is the name of a job store plugin.
 
         :param str|unicode|apscheduler.jobstores.base.BaseJobStore jobstore: job store to be added
         :param str|unicode alias: alias for the job store
         :raises ValueError: if there is already a job store by the given alias
-        """
 
+        """
         with self._jobstores_lock:
             if alias in self._jobstores:
-                raise ValueError('This scheduler already has a job store by the alias of "%s"' % alias)
+                raise ValueError('This scheduler already has a job store by the alias of "%s"' %
+                                 alias)
 
             if isinstance(jobstore, BaseJobStore):
                 self._jobstores[alias] = jobstore
             elif isinstance(jobstore, six.string_types):
-                self._jobstores[alias] = jobstore = self._create_plugin_instance('jobstore', jobstore, jobstore_opts)
+                self._jobstores[alias] = jobstore = self._create_plugin_instance(
+                    'jobstore', jobstore, jobstore_opts)
             else:
                 raise TypeError('Expected a job store instance or a string, got %s instead' %
                                 jobstore.__class__.__name__)
 
-            # Start the job store right away if the scheduler is running
-            if self.running:
+            # Start the job store right away if the scheduler isn't stopped
+            if self.state != STATE_STOPPED:
                 jobstore.start(self, alias)
 
         # Notify listeners that a new job store has been added
         self._dispatch_event(SchedulerEvent(EVENT_JOBSTORE_ADDED, alias))
 
         # Notify the scheduler so it can scan the new job store for jobs
-        if self.running:
+        if self.state != STATE_STOPPED:
             self.wakeup()
 
     def remove_jobstore(self, alias, shutdown=True):
@@ -248,8 +318,8 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         :param str|unicode alias: alias of the job store
         :param bool shutdown: ``True`` to shut down the job store after removing it
-        """
 
+        """
         with self._jobstores_lock:
             jobstore = self._lookup_jobstore(alias)
             del self._jobstores[alias]
@@ -263,17 +333,20 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         """
         add_listener(callback, mask=EVENT_ALL)
 
-        Adds a listener for scheduler events. When a matching event occurs, ``callback`` is executed with the event
-        object as its sole argument. If the ``mask`` parameter is not provided, the callback will receive events of all
-        types.
+        Adds a listener for scheduler events.
+
+        When a matching event  occurs, ``callback`` is executed with the event object as its
+        sole argument. If the ``mask`` parameter is not provided, the callback will receive events
+        of all types.
 
         :param callback: any callable that takes one argument
-        :param int mask: bitmask that indicates which events should be listened to
+        :param int mask: bitmask that indicates which events should be
+            listened to
 
         .. seealso:: :mod:`apscheduler.events`
         .. seealso:: :ref:`scheduler-events`
-        """
 
+        """
         with self._listeners_lock:
             self._listeners.append((callback, mask))
 
@@ -285,47 +358,55 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
                 if callback == cb:
                     del self._listeners[i]
 
-    def add_job(self, func, trigger=None, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined,
-                coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default',
-                executor='default', replace_existing=False, **trigger_args):
+    def add_job(self, func, trigger=None, args=None, kwargs=None, id=None, name=None,
+                misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
+                next_run_time=undefined, jobstore='default', executor='default',
+                replace_existing=False, **trigger_args):
         """
-        add_job(func, trigger=None, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined, \
-            coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default', \
-            executor='default', replace_existing=False, **trigger_args)
+        add_job(func, trigger=None, args=None, kwargs=None, id=None, \
+            name=None, misfire_grace_time=undefined, coalesce=undefined, \
+            max_instances=undefined, next_run_time=undefined, \
+            jobstore='default', executor='default', \
+            replace_existing=False, **trigger_args)
 
         Adds the given job to the job list and wakes up the scheduler if it's already running.
 
-        Any option that defaults to ``undefined`` will be replaced with the corresponding default value when the job is
-        scheduled (which happens when the scheduler is started, or immediately if the scheduler is already running).
+        Any option that defaults to ``undefined`` will be replaced with the corresponding default
+        value when the job is scheduled (which happens when the scheduler is started, or
+        immediately if the scheduler is already running).
 
-        The ``func`` argument can be given either as a callable object or a textual reference in the
-        ``package.module:some.object`` format, where the first half (separated by ``:``) is an importable module and the
-        second half is a reference to the callable object, relative to the module.
+        The ``func`` argument can be given either as a callable object or a textual reference in
+        the ``package.module:some.object`` format, where the first half (separated by ``:``) is an
+        importable module and the second half is a reference to the callable object, relative to
+        the module.
 
         The ``trigger`` argument can either be:
-          #. the alias name of the trigger (e.g. ``date``, ``interval`` or ``cron``), in which case any extra keyword
-             arguments to this method are passed on to the trigger's constructor
+          #. the alias name of the trigger (e.g. ``date``, ``interval`` or ``cron``), in which case
+            any extra keyword arguments to this method are passed on to the trigger's constructor
           #. an instance of a trigger class
 
         :param func: callable (or a textual reference to one) to run at the given time
-        :param str|apscheduler.triggers.base.BaseTrigger trigger: trigger that determines when ``func`` is called
+        :param str|apscheduler.triggers.base.BaseTrigger trigger: trigger that determines when
+            ``func`` is called
         :param list|tuple args: list of positional arguments to call func with
         :param dict kwargs: dict of keyword arguments to call func with
         :param str|unicode id: explicit identifier for the job (for modifying it later)
         :param str|unicode name: textual description of the job
-        :param int misfire_grace_time: seconds after the designated run time that the job is still allowed to be run
-        :param bool coalesce: run once instead of many times if the scheduler determines that the job should be run more
-                              than once in succession
-        :param int max_instances: maximum number of concurrently running instances allowed for this job
-        :param datetime next_run_time: when to first run the job, regardless of the trigger (pass ``None`` to add the
-                                       job as paused)
+        :param int misfire_grace_time: seconds after the designated runtime that the job is still
+            allowed to be run
+        :param bool coalesce: run once instead of many times if the scheduler determines that the
+            job should be run more than once in succession
+        :param int max_instances: maximum number of concurrently running instances allowed for this
+            job
+        :param datetime next_run_time: when to first run the job, regardless of the trigger (pass
+            ``None`` to add the job as paused)
         :param str|unicode jobstore: alias of the job store to store the job in
         :param str|unicode executor: alias of the executor to run the job with
-        :param bool replace_existing: ``True`` to replace an existing job with the same ``id`` (but retain the
-                                      number of runs from the existing one)
+        :param bool replace_existing: ``True`` to replace an existing job with the same ``id``
+            (but retain the number of runs from the existing one)
         :rtype: Job
-        """
 
+        """
         job_kwargs = {
             'trigger': self._create_trigger(trigger, trigger_args),
             'executor': executor,
@@ -339,45 +420,55 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
             'max_instances': max_instances,
             'next_run_time': next_run_time
         }
-        job_kwargs = dict((key, value) for key, value in six.iteritems(job_kwargs) if value is not undefined)
+        job_kwargs = dict((key, value) for key, value in six.iteritems(job_kwargs) if
+                          value is not undefined)
         job = Job(self, **job_kwargs)
 
         # Don't really add jobs to job stores before the scheduler is up and running
         with self._jobstores_lock:
-            if not self.running:
+            if self.state == STATE_STOPPED:
                 self._pending_jobs.append((job, jobstore, replace_existing))
-                self._logger.info('Adding job tentatively -- it will be properly scheduled when the scheduler starts')
+                self._logger.info('Adding job tentatively -- it will be properly scheduled when '
+                                  'the scheduler starts')
             else:
-                self._real_add_job(job, jobstore, replace_existing, True)
+                self._real_add_job(job, jobstore, replace_existing)
 
         return job
 
-    def scheduled_job(self, trigger, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined,
-                      coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default',
-                      executor='default', **trigger_args):
+    def scheduled_job(self, trigger, args=None, kwargs=None, id=None, name=None,
+                      misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
+                      next_run_time=undefined, jobstore='default', executor='default',
+                      **trigger_args):
         """
-        scheduled_job(trigger, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined, \
-            coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default', \
+        scheduled_job(trigger, args=None, kwargs=None, id=None, \
+            name=None, misfire_grace_time=undefined, \
+            coalesce=undefined, max_instances=undefined, \
+            next_run_time=undefined, jobstore='default', \
             executor='default',**trigger_args)
 
-        A decorator version of :meth:`add_job`, except that ``replace_existing`` is always ``True``.
+        A decorator version of :meth:`add_job`, except that ``replace_existing`` is always
+        ``True``.
+
+        .. important:: The ``id`` argument must be given if scheduling a job in a persistent job
+        store. The scheduler cannot, however, enforce this requirement.
 
-        .. important:: The ``id`` argument must be given if scheduling a job in a persistent job store. The scheduler
-           cannot, however, enforce this requirement.
         """
-
         def inner(func):
-            self.add_job(func, trigger, args, kwargs, id, name, misfire_grace_time, coalesce, max_instances,
-                         next_run_time, jobstore, executor, True, **trigger_args)
+            self.add_job(func, trigger, args, kwargs, id, name, misfire_grace_time, coalesce,
+                         max_instances, next_run_time, jobstore, executor, True, **trigger_args)
             return func
         return inner
 
     def modify_job(self, job_id, jobstore=None, **changes):
         """
-        Modifies the properties of a single job. Modifications are passed to this method as extra keyword arguments.
+        Modifies the properties of a single job.
+
+        Modifications are passed to this method as extra keyword arguments.
 
         :param str|unicode job_id: the identifier of the job
         :param str|unicode jobstore: alias of the job store that contains the job
+        :return Job: the relevant job instance
+
         """
         with self._jobstores_lock:
             job, jobstore = self._lookup_job(job_id, jobstore)
@@ -388,22 +479,27 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         self._dispatch_event(JobEvent(EVENT_JOB_MODIFIED, job_id, jobstore))
 
         # Wake up the scheduler since the job's next run time may have been changed
+        if self.state == STATE_RUNNING:
             self.wakeup()
 
+        return job
+
     def reschedule_job(self, job_id, jobstore=None, trigger=None, **trigger_args):
         """
         Constructs a new trigger for a job and updates its next run time.
+
         Extra keyword arguments are passed directly to the trigger's constructor.
 
         :param str|unicode job_id: the identifier of the job
         :param str|unicode jobstore: alias of the job store that contains the job
         :param trigger: alias of the trigger type or a trigger instance
-        """
+        :return Job: the relevant job instance
 
+        """
         trigger = self._create_trigger(trigger, trigger_args)
         now = datetime.now(self.timezone)
         next_run_time = trigger.get_next_fire_time(None, now)
-        self.modify_job(job_id, jobstore, trigger=trigger, next_run_time=next_run_time)
+        return self.modify_job(job_id, jobstore, trigger=trigger, next_run_time=next_run_time)
 
     def pause_job(self, job_id, jobstore=None):
         """
@@ -411,9 +507,10 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         :param str|unicode job_id: the identifier of the job
         :param str|unicode jobstore: alias of the job store that contains the job
-        """
+        :return Job: the relevant job instance
 
-        self.modify_job(job_id, jobstore, next_run_time=None)
+        """
+        return self.modify_job(job_id, jobstore, next_run_time=None)
 
     def resume_job(self, job_id, jobstore=None):
         """
@@ -421,38 +518,44 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         :param str|unicode job_id: the identifier of the job
         :param str|unicode jobstore: alias of the job store that contains the job
-        """
+        :return Job|None: the relevant job instance if the job was rescheduled, or ``None`` if no
+            next run time could be calculated and the job was removed
 
+        """
         with self._jobstores_lock:
             job, jobstore = self._lookup_job(job_id, jobstore)
             now = datetime.now(self.timezone)
             next_run_time = job.trigger.get_next_fire_time(None, now)
             if next_run_time:
-                self.modify_job(job_id, jobstore, next_run_time=next_run_time)
+                return self.modify_job(job_id, jobstore, next_run_time=next_run_time)
             else:
                 self.remove_job(job.id, jobstore)
 
     def get_jobs(self, jobstore=None, pending=None):
         """
-        Returns a list of pending jobs (if the scheduler hasn't been started yet) and scheduled jobs, either from a
-        specific job store or from all of them.
+        Returns a list of pending jobs (if the scheduler hasn't been started yet) and scheduled
+        jobs, either from a specific job store or from all of them.
+
+        If the scheduler has not been started yet, only pending jobs can be returned because the
+        job stores haven't been started yet either.
 
         :param str|unicode jobstore: alias of the job store
-        :param bool pending: ``False`` to leave out pending jobs (jobs that are waiting for the scheduler start to be
-                             added to their respective job stores), ``True`` to only include pending jobs, anything else
-                             to return both
+        :param bool pending: **DEPRECATED**
         :rtype: list[Job]
+
         """
+        if pending is not None:
+            warnings.warn('The "pending" option is deprecated -- get_jobs() always returns '
+                          'pending jobs if the scheduler has been started and scheduled jobs '
+                          'otherwise', DeprecationWarning)
 
         with self._jobstores_lock:
             jobs = []
-
-            if pending is not False:
+            if self.state == STATE_STOPPED:
                 for job, alias, replace_existing in self._pending_jobs:
                     if jobstore is None or alias == jobstore:
                         jobs.append(job)
-
-            if pending is not True:
+            else:
                 for alias, store in six.iteritems(self._jobstores):
                     if jobstore is None or alias == jobstore:
                         jobs.extend(store.get_all_jobs())
@@ -467,8 +570,8 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         :param str|unicode jobstore: alias of the job store that most likely contains the job
         :return: the Job by the given ID, or ``None`` if it wasn't found
         :rtype: Job
-        """
 
+        """
         with self._jobstores_lock:
             try:
                 return self._lookup_job(job_id, jobstore)[0]
@@ -482,32 +585,35 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         :param str|unicode job_id: the identifier of the job
         :param str|unicode jobstore: alias of the job store that contains the job
         :raises JobLookupError: if the job was not found
-        """
 
+        """
+        jobstore_alias = None
         with self._jobstores_lock:
+            if self.state == STATE_STOPPED:
                 # Check if the job is among the pending jobs
-            for i, (job, jobstore_alias, replace_existing) in enumerate(self._pending_jobs):
-                if job.id == job_id:
+                if self.state == STATE_STOPPED:
+                    for i, (job, alias, replace_existing) in enumerate(self._pending_jobs):
+                        if job.id == job_id and jobstore in (None, alias):
                             del self._pending_jobs[i]
-                    jobstore = jobstore_alias
+                            jobstore_alias = alias
                             break
             else:
-                # Otherwise, try to remove it from each store until it succeeds or we run out of stores to check
+                # Otherwise, try to remove it from each store until it succeeds or we run out of
+                # stores to check
                 for alias, store in six.iteritems(self._jobstores):
                     if jobstore in (None, alias):
                         try:
                             store.remove_job(job_id)
+                            jobstore_alias = alias
+                            break
                         except JobLookupError:
                             continue
 
-                        jobstore = alias
-                        break
-
-        if jobstore is None:
+        if jobstore_alias is None:
             raise JobLookupError(job_id)
 
         # Notify listeners that a job has been removed
-        event = JobEvent(EVENT_JOB_REMOVED, job_id, jobstore)
+        event = JobEvent(EVENT_JOB_REMOVED, job_id, jobstore_alias)
         self._dispatch_event(event)
 
         self._logger.info('Removed job %s', job_id)
@@ -517,14 +623,16 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         Removes all jobs from the specified job store, or all job stores if none is given.
 
         :param str|unicode jobstore: alias of the job store
-        """
 
+        """
         with self._jobstores_lock:
+            if self.state == STATE_STOPPED:
                 if jobstore:
-                self._pending_jobs = [pending for pending in self._pending_jobs if pending[1] != jobstore]
+                    self._pending_jobs = [pending for pending in self._pending_jobs if
+                                          pending[1] != jobstore]
                 else:
                     self._pending_jobs = []
-
+            else:
                 for alias, store in six.iteritems(self._jobstores):
                     if jobstore in (None, alias):
                         store.remove_all_jobs()
@@ -535,29 +643,34 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         """
         print_jobs(jobstore=None, out=sys.stdout)
 
-        Prints out a textual listing of all jobs currently scheduled on either all job stores or just a specific one.
+        Prints out a textual listing of all jobs currently scheduled on either all job stores or
+        just a specific one.
 
         :param str|unicode jobstore: alias of the job store, ``None`` to list jobs from all stores
-        :param file out: a file-like object to print to (defaults to **sys.stdout** if nothing is given)
-        """
+        :param file out: a file-like object to print to (defaults to  **sys.stdout** if nothing is
+            given)
 
+        """
         out = out or sys.stdout
         with self._jobstores_lock:
+            if self.state == STATE_STOPPED:
+                print(u'Pending jobs:', file=out)
                 if self._pending_jobs:
-                print(six.u('Pending jobs:'), file=out)
                     for job, jobstore_alias, replace_existing in self._pending_jobs:
                         if jobstore in (None, jobstore_alias):
-                        print(six.u('    %s') % job, file=out)
-
-            for alias, store in six.iteritems(self._jobstores):
+                            print(u'    %s' % job, file=out)
+                else:
+                    print(u'    No pending jobs', file=out)
+            else:
+                for alias, store in sorted(six.iteritems(self._jobstores)):
                     if jobstore in (None, alias):
-                    print(six.u('Jobstore %s:') % alias, file=out)
+                        print(u'Jobstore %s:' % alias, file=out)
                         jobs = store.get_all_jobs()
                         if jobs:
                             for job in jobs:
-                            print(six.u('    %s') % job, file=out)
+                                print(u'    %s' % job, file=out)
                         else:
-                        print(six.u('    No scheduled jobs'), file=out)
+                            print(u'    No scheduled jobs', file=out)
 
     @abstractmethod
     def wakeup(self):
@@ -574,6 +687,7 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         # Set general options
         self._logger = maybe_ref(config.pop('logger', None)) or getLogger('apscheduler.scheduler')
         self.timezone = astimezone(config.pop('timezone', None)) or get_localzone()
+        self.jobstore_retry_interval = float(config.pop('jobstore_retry_interval', 10))
 
         # Set the job defaults
         job_defaults = config.get('job_defaults', {})
@@ -597,12 +711,15 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
                     cls = maybe_ref(executor_class)
                     executor = cls(**value)
                 else:
-                    raise ValueError('Cannot create executor "%s" -- either "type" or "class" must be defined' % alias)
+                    raise ValueError(
+                        'Cannot create executor "%s" -- either "type" or "class" must be defined' %
+                        alias)
 
                 self.add_executor(executor, alias)
             else:
-                raise TypeError("Expected executor instance or dict for executors['%s'], got %s instead" % (
-                    alias, value.__class__.__name__))
+                raise TypeError(
+                    "Expected executor instance or dict for executors['%s'], got %s instead" %
+                    (alias, value.__class__.__name__))
 
         # Configure job stores
         self._jobstores.clear()
@@ -618,31 +735,33 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
                     cls = maybe_ref(jobstore_class)
                     jobstore = cls(**value)
                 else:
-                    raise ValueError('Cannot create job store "%s" -- either "type" or "class" must be defined' % alias)
+                    raise ValueError(
+                        'Cannot create job store "%s" -- either "type" or "class" must be '
+                        'defined' % alias)
 
                 self.add_jobstore(jobstore, alias)
             else:
-                raise TypeError("Expected job store instance or dict for jobstores['%s'], got %s instead" % (
-                    alias, value.__class__.__name__))
+                raise TypeError(
+                    "Expected job store instance or dict for jobstores['%s'], got %s instead" %
+                    (alias, value.__class__.__name__))
 
     def _create_default_executor(self):
         """Creates a default executor store, specific to the particular scheduler type."""
-
         return ThreadPoolExecutor()
 
     def _create_default_jobstore(self):
         """Creates a default job store, specific to the particular scheduler type."""
-
         return MemoryJobStore()
 
     def _lookup_executor(self, alias):
         """
-        Returns the executor instance by the given name from the list of executors that were added to this scheduler.
+        Returns the executor instance by the given name from the list of executors that were added
+        to this scheduler.
 
         :type alias: str
         :raises KeyError: if no executor by the given alias is not found
-        """
 
+        """
         try:
             return self._executors[alias]
         except KeyError:
@@ -650,12 +769,13 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
     def _lookup_jobstore(self, alias):
         """
-        Returns the job store instance by the given name from the list of job stores that were added to this scheduler.
+        Returns the job store instance by the given name from the list of job stores that were
+        added to this scheduler.
 
         :type alias: str
         :raises KeyError: if no job store by the given alias is not found
-        """
 
+        """
         try:
             return self._jobstores[alias]
         except KeyError:
@@ -667,15 +787,17 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
         :type job_id: str
         :param str jobstore_alias: alias of a job store to look in
-        :return tuple[Job, str]: a tuple of job, jobstore alias (jobstore alias is None in case of a pending job)
+        :return tuple[Job, str]: a tuple of job, jobstore alias (jobstore alias is None in case of
+            a pending job)
         :raises JobLookupError: if no job by the given ID is found.
-        """
 
+        """
+        if self.state == STATE_STOPPED:
             # Check if the job is among the pending jobs
             for job, alias, replace_existing in self._pending_jobs:
                 if job.id == job_id:
                     return job, None
-
+        else:
             # Look in all job stores
             for alias, store in six.iteritems(self._jobstores):
                 if jobstore_alias in (None, alias):
@@ -690,8 +812,8 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         Dispatches the given event to interested listeners.
 
         :param SchedulerEvent event: the event to send
-        """
 
+        """
         with self._listeners_lock:
             listeners = tuple(self._listeners)
 
@@ -699,16 +821,16 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
             if event.code & mask:
                 try:
                     cb(event)
-                except:
+                except BaseException:
                     self._logger.exception('Error notifying listener')
 
-    def _real_add_job(self, job, jobstore_alias, replace_existing, wakeup):
+    def _real_add_job(self, job, jobstore_alias, replace_existing):
         """
         :param Job job: the job to add
-        :param bool replace_existing: ``True`` to use update_job() in case the job already exists in the store
-        :param bool wakeup: ``True`` to wake up the scheduler after adding the job
-        """
+        :param bool replace_existing: ``True`` to use update_job() in case the job already exists
+            in the store
 
+        """
         # Fill in undefined values with defaults
         replacements = {}
         for key, value in six.iteritems(self._job_defaults):
@@ -743,12 +865,11 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         self._logger.info('Added job "%s" to job store "%s"', job.name, jobstore_alias)
 
         # Notify the scheduler about the new job
-        if wakeup:
+        if self.state == STATE_RUNNING:
             self.wakeup()
 
     def _create_plugin_instance(self, type_, alias, constructor_kwargs):
         """Creates an instance of the given plugin type, loading the plugin first if necessary."""
-
         plugin_container, class_container, base_class = {
             'trigger': (self._trigger_plugins, self._trigger_classes, BaseTrigger),
             'jobstore': (self._jobstore_plugins, self._jobstore_classes, BaseJobStore),
@@ -761,7 +882,8 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
             if alias in plugin_container:
                 plugin_cls = class_container[alias] = plugin_container[alias].load()
                 if not issubclass(plugin_cls, base_class):
-                    raise TypeError('The {0} entry point does not point to a {0} class'.format(type_))
+                    raise TypeError('The {0} entry point does not point to a {0} class'.
+                                    format(type_))
             else:
                 raise LookupError('No {0} by the name "{1}" was found'.format(type_, alias))
 
@@ -773,7 +895,8 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
         elif trigger is None:
             trigger = 'date'
         elif not isinstance(trigger, six.string_types):
-            raise TypeError('Expected a trigger instance or string, got %s instead' % trigger.__class__.__name__)
+            raise TypeError('Expected a trigger instance or string, got %s instead' %
+                            trigger.__class__.__name__)
 
         # Use the scheduler's time zone if nothing else is specified
         trigger_args.setdefault('timezone', self.timezone)
@@ -783,29 +906,48 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
 
     def _create_lock(self):
         """Creates a reentrant lock object."""
-
         return RLock()
 
     def _process_jobs(self):
         """
-        Iterates through jobs in every jobstore, starts jobs that are due and figures out how long to wait for the next
-        round.
+        Iterates through jobs in every jobstore, starts jobs that are due and figures out how long
+        to wait for the next round.
+
+        If the ``get_due_jobs()`` call raises an exception, a new wakeup is scheduled in at least
+        ``jobstore_retry_interval`` seconds.
+
         """
+        if self.state == STATE_PAUSED:
+            self._logger.debug('Scheduler is paused -- not processing jobs')
+            return None
 
         self._logger.debug('Looking for jobs to run')
         now = datetime.now(self.timezone)
         next_wakeup_time = None
+        events = []
 
         with self._jobstores_lock:
             for jobstore_alias, jobstore in six.iteritems(self._jobstores):
-                for job in jobstore.get_due_jobs(now):
+                try:
+                    due_jobs = jobstore.get_due_jobs(now)
+                except Exception as e:
+                    # Schedule a wakeup at least in jobstore_retry_interval seconds
+                    self._logger.warning('Error getting due jobs from job store %r: %s',
+                                         jobstore_alias, e)
+                    retry_wakeup_time = now + timedelta(seconds=self.jobstore_retry_interval)
+                    if not next_wakeup_time or next_wakeup_time > retry_wakeup_time:
+                        next_wakeup_time = retry_wakeup_time
+
+                    continue
+
+                for job in due_jobs:
                     # Look up the job's executor
                     try:
                         executor = self._lookup_executor(job.executor)
-                    except:
+                    except BaseException:
                         self._logger.error(
-                            'Executor lookup ("%s") failed for job "%s" -- removing it from the job store',
-                            job.executor, job)
+                            'Executor lookup ("%s") failed for job "%s" -- removing it from the '
+                            'job store', job.executor, job)
                         self.remove_job(job.id, jobstore_alias)
                         continue
 
@@ -816,12 +958,21 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
                             executor.submit_job(job, run_times)
                         except MaxInstancesReachedError:
                             self._logger.warning(
-                                'Execution of job "%s" skipped: maximum number of running instances reached (%d)',
-                                job, job.max_instances)
-                        except:
-                            self._logger.exception('Error submitting job "%s" to executor "%s"', job, job.executor)
+                                'Execution of job "%s" skipped: maximum number of running '
+                                'instances reached (%d)', job, job.max_instances)
+                            event = JobSubmissionEvent(EVENT_JOB_MAX_INSTANCES, job.id,
+                                                       jobstore_alias, run_times)
+                            events.append(event)
+                        except BaseException:
+                            self._logger.exception('Error submitting job "%s" to executor "%s"',
+                                                   job, job.executor)
+                        else:
+                            event = JobSubmissionEvent(EVENT_JOB_SUBMITTED, job.id, jobstore_alias,
+                                                       run_times)
+                            events.append(event)
 
-                        # Update the job if it has a next execution time. Otherwise remove it from the job store.
+                        # Update the job if it has a next execution time.
+                        # Otherwise remove it from the job store.
                         job_next_run = job.trigger.get_next_fire_time(run_times[-1], now)
                         if job_next_run:
                             job._modify(next_run_time=job_next_run)
@@ -829,17 +980,27 @@ class BaseScheduler(six.with_metaclass(ABCMeta)):
                         else:
                             self.remove_job(job.id, jobstore_alias)
 
-                # Set a new next wakeup time if there isn't one yet or the jobstore has an even earlier one
+                # Set a new next wakeup time if there isn't one yet or
+                # the jobstore has an even earlier one
                 jobstore_next_run_time = jobstore.get_next_run_time()
-                if jobstore_next_run_time and (next_wakeup_time is None or jobstore_next_run_time < next_wakeup_time):
-                    next_wakeup_time = jobstore_next_run_time
+                if jobstore_next_run_time and (next_wakeup_time is None or
+                                               jobstore_next_run_time < next_wakeup_time):
+                    next_wakeup_time = jobstore_next_run_time.astimezone(self.timezone)
+
+        # Dispatch collected events
+        for event in events:
+            self._dispatch_event(event)
 
         # Determine the delay until this method should be called again
-        if next_wakeup_time is not None:
-            wait_seconds = max(timedelta_seconds(next_wakeup_time - now), 0)
-            self._logger.debug('Next wakeup is due at %s (in %f seconds)', next_wakeup_time, wait_seconds)
-        else:
+        if self.state == STATE_PAUSED:
+            wait_seconds = None
+            self._logger.debug('Scheduler is paused; waiting until resume() is called')
+        elif next_wakeup_time is None:
             wait_seconds = None
             self._logger.debug('No jobs; waiting until a job is added')
+        else:
+            wait_seconds = max(timedelta_seconds(next_wakeup_time - now), 0)
+            self._logger.debug('Next wakeup is due at %s (in %f seconds)', next_wakeup_time,
+                               wait_seconds)
 
         return wait_seconds

lib/apscheduler/schedulers/blocking.py

@@ -1,21 +1,21 @@
 from __future__ import absolute_import
+
 from threading import Event
 
-from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.schedulers.base import BaseScheduler, STATE_STOPPED
+from apscheduler.util import TIMEOUT_MAX
 
 
 class BlockingScheduler(BaseScheduler):
     """
-    A scheduler that runs in the foreground (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will block).
+    A scheduler that runs in the foreground
+    (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will block).
     """
-
-    MAX_WAIT_TIME = 4294967  # Maximum value accepted by Event.wait() on Windows
-
     _event = None
 
-    def start(self):
-        super(BlockingScheduler, self).start()
+    def start(self, *args, **kwargs):
         self._event = Event()
+        super(BlockingScheduler, self).start(*args, **kwargs)
         self._main_loop()
 
     def shutdown(self, wait=True):
@@ -23,10 +23,11 @@ class BlockingScheduler(BaseScheduler):
         self._event.set()
 
     def _main_loop(self):
-        while self.running:
-            wait_seconds = self._process_jobs()
-            self._event.wait(wait_seconds if wait_seconds is not None else self.MAX_WAIT_TIME)
+        wait_seconds = TIMEOUT_MAX
+        while self.state != STATE_STOPPED:
+            self._event.wait(wait_seconds)
             self._event.clear()
+            wait_seconds = self._process_jobs()
 
     def wakeup(self):
         self._event.set()

lib/apscheduler/schedulers/gevent.py

@@ -16,14 +16,14 @@ class GeventScheduler(BlockingScheduler):
 
     _greenlet = None
 
-    def start(self):
-        BaseScheduler.start(self)
+    def start(self, *args, **kwargs):
         self._event = Event()
+        BaseScheduler.start(self, *args, **kwargs)
         self._greenlet = gevent.spawn(self._main_loop)
         return self._greenlet
 
-    def shutdown(self, wait=True):
-        super(GeventScheduler, self).shutdown(wait)
+    def shutdown(self, *args, **kwargs):
+        super(GeventScheduler, self).shutdown(*args, **kwargs)
         self._greenlet.join()
         del self._greenlet
 

lib/apscheduler/schedulers/qt.py

@@ -4,7 +4,7 @@ from apscheduler.schedulers.base import BaseScheduler
 
 try:
     from PyQt5.QtCore import QObject, QTimer
-except ImportError:  # pragma: nocover
+except (ImportError, RuntimeError):  # pragma: nocover
     try:
         from PyQt4.QtCore import QObject, QTimer
     except ImportError:
@@ -19,12 +19,8 @@ class QtScheduler(BaseScheduler):
 
     _timer = None
 
-    def start(self):
-        super(QtScheduler, self).start()
-        self.wakeup()
-
-    def shutdown(self, wait=True):
-        super(QtScheduler, self).shutdown(wait)
+    def shutdown(self, *args, **kwargs):
+        super(QtScheduler, self).shutdown(*args, **kwargs)
         self._stop_timer()
 
     def _start_timer(self, wait_seconds):

lib/apscheduler/schedulers/tornado.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+
 from datetime import timedelta
 from functools import wraps
 
@@ -22,6 +23,8 @@ class TornadoScheduler(BaseScheduler):
     """
     A scheduler that runs on a Tornado IOLoop.
 
+    The default executor can run jobs based on native coroutines (``async def``).
+
     =========== ===============================================================
     ``io_loop`` Tornado IOLoop instance to use (defaults to the global IO loop)
     =========== ===============================================================
@@ -30,10 +33,6 @@ class TornadoScheduler(BaseScheduler):
     _ioloop = None
     _timeout = None
 
-    def start(self):
-        super(TornadoScheduler, self).start()
-        self.wakeup()
-
     @run_in_ioloop
     def shutdown(self, wait=True):
         super(TornadoScheduler, self).shutdown(wait)
@@ -53,6 +52,10 @@ class TornadoScheduler(BaseScheduler):
             self._ioloop.remove_timeout(self._timeout)
             del self._timeout
 
+    def _create_default_executor(self):
+        from apscheduler.executors.tornado import TornadoExecutor
+        return TornadoExecutor()
+
     @run_in_ioloop
     def wakeup(self):
         self._stop_timer()

lib/apscheduler/schedulers/twisted.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+
 from functools import wraps
 
 from apscheduler.schedulers.base import BaseScheduler
@@ -35,10 +36,6 @@ class TwistedScheduler(BaseScheduler):
         self._reactor = maybe_ref(config.pop('reactor', default_reactor))
         super(TwistedScheduler, self)._configure(config)
 
-    def start(self):
-        super(TwistedScheduler, self).start()
-        self.wakeup()
-
     @run_in_reactor
     def shutdown(self, wait=True):
         super(TwistedScheduler, self).shutdown(wait)

lib/apscheduler/triggers/base.py

@@ -1,4 +1,6 @@
 from abc import ABCMeta, abstractmethod
+from datetime import timedelta
+import random
 
 import six
 
@@ -6,11 +8,41 @@ import six
 class BaseTrigger(six.with_metaclass(ABCMeta)):
     """Abstract base class that defines the interface that every trigger must implement."""
 
+    __slots__ = ()
+
     @abstractmethod
     def get_next_fire_time(self, previous_fire_time, now):
         """
-        Returns the next datetime to fire on, If no such datetime can be calculated, returns ``None``.
+        Returns the next datetime to fire on, If no such datetime can be calculated, returns
+        ``None``.
 
         :param datetime.datetime previous_fire_time: the previous time the trigger was fired
         :param datetime.datetime now: current datetime
         """
+
+    def _apply_jitter(self, next_fire_time, jitter, now):
+        """
+        Randomize ``next_fire_time`` by adding or subtracting a random value (the jitter). If the
+        resulting datetime is in the past, returns the initial ``next_fire_time`` without jitter.
+
+        ``next_fire_time - jitter <= result <= next_fire_time + jitter``
+
+        :param datetime.datetime|None next_fire_time: next fire time without jitter applied. If
+            ``None``, returns ``None``.
+        :param int|None jitter: maximum number of seconds to add or subtract to
+            ``next_fire_time``. If ``None`` or ``0``, returns ``next_fire_time``
+        :param datetime.datetime now: current datetime
+        :return datetime.datetime|None: next fire time with a jitter.
+        """
+        if next_fire_time is None or not jitter:
+            return next_fire_time
+
+        next_fire_time_with_jitter = next_fire_time + timedelta(
+                seconds=random.uniform(-jitter, jitter))
+
+        if next_fire_time_with_jitter < now:
+            # Next fire time with jitter is in the past.
+            # Ignore jitter to avoid false misfire.
+            return next_fire_time
+
+        return next_fire_time_with_jitter

lib/apscheduler/triggers/combining.py

@@ -0,0 +1,95 @@
+from apscheduler.triggers.base import BaseTrigger
+from apscheduler.util import obj_to_ref, ref_to_obj
+
+
+class BaseCombiningTrigger(BaseTrigger):
+    __slots__ = ('triggers', 'jitter')
+
+    def __init__(self, triggers, jitter=None):
+        self.triggers = triggers
+        self.jitter = jitter
+
+    def __getstate__(self):
+        return {
+            'version': 1,
+            'triggers': [(obj_to_ref(trigger.__class__), trigger.__getstate__())
+                         for trigger in self.triggers],
+            'jitter': self.jitter
+        }
+
+    def __setstate__(self, state):
+        if state.get('version', 1) > 1:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 1 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.jitter = state['jitter']
+        self.triggers = []
+        for clsref, state in state['triggers']:
+            cls = ref_to_obj(clsref)
+            trigger = cls.__new__(cls)
+            trigger.__setstate__(state)
+            self.triggers.append(trigger)
+
+    def __repr__(self):
+        return '<{}({}{})>'.format(self.__class__.__name__, self.triggers,
+                                   ', jitter={}'.format(self.jitter) if self.jitter else '')
+
+
+class AndTrigger(BaseCombiningTrigger):
+    """
+    Always returns the earliest next fire time that all the given triggers can agree on.
+    The trigger is considered to be finished when any of the given triggers has finished its
+    schedule.
+
+    Trigger alias: ``and``
+
+    :param list triggers: triggers to combine
+    :param int|None jitter: advance or delay the job execution by ``jitter`` seconds at most.
+    """
+
+    __slots__ = ()
+
+    def get_next_fire_time(self, previous_fire_time, now):
+        while True:
+            fire_times = [trigger.get_next_fire_time(previous_fire_time, now)
+                          for trigger in self.triggers]
+            if None in fire_times:
+                return None
+            elif min(fire_times) == max(fire_times):
+                return self._apply_jitter(fire_times[0], self.jitter, now)
+            else:
+                now = max(fire_times)
+
+    def __str__(self):
+        return 'and[{}]'.format(', '.join(str(trigger) for trigger in self.triggers))
+
+
+class OrTrigger(BaseCombiningTrigger):
+    """
+    Always returns the earliest next fire time produced by any of the given triggers.
+    The trigger is considered finished when all the given triggers have finished their schedules.
+
+    Trigger alias: ``or``
+
+    :param list triggers: triggers to combine
+    :param int|None jitter: advance or delay the job execution by ``jitter`` seconds at most.
+
+    .. note:: Triggers that depends on the previous fire time, such as the interval trigger, may
+        seem to behave strangely since they are always passed the previous fire time produced by
+        any of the given triggers.
+    """
+
+    __slots__ = ()
+
+    def get_next_fire_time(self, previous_fire_time, now):
+        fire_times = [trigger.get_next_fire_time(previous_fire_time, now)
+                      for trigger in self.triggers]
+        fire_times = [fire_time for fire_time in fire_times if fire_time is not None]
+        if fire_times:
+            return self._apply_jitter(min(fire_times), self.jitter, now)
+        else:
+            return None
+
+    def __str__(self):
+        return 'or[{}]'.format(', '.join(str(trigger) for trigger in self.triggers))

lib/apscheduler/triggers/cron/__init__.py

@@ -4,13 +4,15 @@ from tzlocal import get_localzone
 import six
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.triggers.cron.fields import BaseField, WeekField, DayOfMonthField, DayOfWeekField, DEFAULT_VALUES
+from apscheduler.triggers.cron.fields import (
+    BaseField, MonthField, WeekField, DayOfMonthField, DayOfWeekField, DEFAULT_VALUES)
 from apscheduler.util import datetime_ceil, convert_to_datetime, datetime_repr, astimezone
 
 
 class CronTrigger(BaseTrigger):
     """
-    Triggers when current time matches all specified time constraints, similarly to how the UNIX cron scheduler works.
+    Triggers when current time matches all specified time constraints,
+    similarly to how the UNIX cron scheduler works.
 
     :param int|str year: 4-digit year
     :param int|str month: month (1-12)
@@ -22,8 +24,9 @@ class CronTrigger(BaseTrigger):
     :param int|str second: second (0-59)
     :param datetime|str start_date: earliest possible date/time to trigger on (inclusive)
     :param datetime|str end_date: latest possible date/time to trigger on (inclusive)
-    :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations
-                                         (defaults to scheduler timezone)
+    :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations (defaults
+        to scheduler timezone)
+    :param int|None jitter: advance or delay the job execution by ``jitter`` seconds at most.
 
     .. note:: The first weekday is always **monday**.
     """
@@ -31,7 +34,7 @@ class CronTrigger(BaseTrigger):
     FIELD_NAMES = ('year', 'month', 'day', 'week', 'day_of_week', 'hour', 'minute', 'second')
     FIELDS_MAP = {
         'year': BaseField,
-        'month': BaseField,
+        'month': MonthField,
         'week': WeekField,
         'day': DayOfMonthField,
         'day_of_week': DayOfWeekField,
@@ -40,15 +43,16 @@ class CronTrigger(BaseTrigger):
         'second': BaseField
     }
 
-    __slots__ = 'timezone', 'start_date', 'end_date', 'fields'
+    __slots__ = 'timezone', 'start_date', 'end_date', 'fields', 'jitter'
 
-    def __init__(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None, minute=None,
-                 second=None, start_date=None, end_date=None, timezone=None):
+    def __init__(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None,
+                 minute=None, second=None, start_date=None, end_date=None, timezone=None,
+                 jitter=None):
         if timezone:
             self.timezone = astimezone(timezone)
-        elif start_date and start_date.tzinfo:
+        elif isinstance(start_date, datetime) and start_date.tzinfo:
             self.timezone = start_date.tzinfo
-        elif end_date and end_date.tzinfo:
+        elif isinstance(end_date, datetime) and end_date.tzinfo:
             self.timezone = end_date.tzinfo
         else:
             self.timezone = get_localzone()
@@ -56,6 +60,8 @@ class CronTrigger(BaseTrigger):
         self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date')
         self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date')
 
+        self.jitter = jitter
+
         values = dict((key, value) for (key, value) in six.iteritems(locals())
                       if key in self.FIELD_NAMES and value is not None)
         self.fields = []
@@ -76,13 +82,35 @@ class CronTrigger(BaseTrigger):
             field = field_class(field_name, exprs, is_default)
             self.fields.append(field)
 
+    @classmethod
+    def from_crontab(cls, expr, timezone=None):
+        """
+        Create a :class:`~CronTrigger` from a standard crontab expression.
+
+        See https://en.wikipedia.org/wiki/Cron for more information on the format accepted here.
+
+        :param expr: minute, hour, day of month, month, day of week
+        :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations (
+            defaults to scheduler timezone)
+        :return: a :class:`~CronTrigger` instance
+
+        """
+        values = expr.split()
+        if len(values) != 5:
+            raise ValueError('Wrong number of fields; got {}, expected 5'.format(len(values)))
+
+        return cls(minute=values[0], hour=values[1], day=values[2], month=values[3],
+                   day_of_week=values[4], timezone=timezone)
+
     def _increment_field_value(self, dateval, fieldnum):
         """
-        Increments the designated field and resets all less significant fields to their minimum values.
+        Increments the designated field and resets all less significant fields to their minimum
+        values.
 
         :type dateval: datetime
         :type fieldnum: int
-        :return: a tuple containing the new date, and the number of the field that was actually incremented
+        :return: a tuple containing the new date, and the number of the field that was actually
+            incremented
         :rtype: tuple
         """
 
@@ -128,12 +156,13 @@ class CronTrigger(BaseTrigger):
                 else:
                     values[field.name] = new_value
 
-        difference = datetime(**values) - dateval.replace(tzinfo=None)
-        return self.timezone.normalize(dateval + difference)
+        return self.timezone.localize(datetime(**values))
 
     def get_next_fire_time(self, previous_fire_time, now):
         if previous_fire_time:
-            start_date = max(now, previous_fire_time + timedelta(microseconds=1))
+            start_date = min(now, previous_fire_time + timedelta(microseconds=1))
+            if start_date == previous_fire_time:
+                start_date += timedelta(microseconds=1)
         else:
             start_date = max(now, self.start_date) if self.start_date else now
 
@@ -163,8 +192,36 @@ class CronTrigger(BaseTrigger):
                 return None
 
         if fieldnum >= 0:
+            if self.jitter is not None:
+                next_date = self._apply_jitter(next_date, self.jitter, now)
             return next_date
 
+    def __getstate__(self):
+        return {
+            'version': 2,
+            'timezone': self.timezone,
+            'start_date': self.start_date,
+            'end_date': self.end_date,
+            'fields': self.fields,
+            'jitter': self.jitter,
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 2:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 2 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.timezone = state['timezone']
+        self.start_date = state['start_date']
+        self.end_date = state['end_date']
+        self.fields = state['fields']
+        self.jitter = state.get('jitter')
+
     def __str__(self):
         options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default]
         return 'cron[%s]' % (', '.join(options))
@@ -172,5 +229,11 @@ class CronTrigger(BaseTrigger):
     def __repr__(self):
         options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default]
         if self.start_date:
-            options.append("start_date='%s'" % datetime_repr(self.start_date))
-        return '<%s (%s)>' % (self.__class__.__name__, ', '.join(options))
+            options.append("start_date=%r" % datetime_repr(self.start_date))
+        if self.end_date:
+            options.append("end_date=%r" % datetime_repr(self.end_date))
+        if self.jitter:
+            options.append('jitter=%s' % self.jitter)
+
+        return "<%s (%s, timezone='%s')>" % (
+            self.__class__.__name__, ', '.join(options), self.timezone)

lib/apscheduler/triggers/cron/expressions.py

@@ -1,17 +1,16 @@
-"""
-This module contains the expressions applicable for CronTrigger's fields.
-"""
+"""This module contains the expressions applicable for CronTrigger's fields."""
 
 from calendar import monthrange
 import re
 
 from apscheduler.util import asint
 
-__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression', 'WeekdayPositionExpression',
-           'LastDayOfMonthExpression')
+__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression',
+           'WeekdayPositionExpression', 'LastDayOfMonthExpression')
 
 
 WEEKDAYS = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
+MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
 
 
 class AllExpression(object):
@@ -22,6 +21,14 @@ class AllExpression(object):
         if self.step == 0:
             raise ValueError('Increment must be higher than 0')
 
+    def validate_range(self, field_name):
+        from apscheduler.triggers.cron.fields import MIN_VALUES, MAX_VALUES
+
+        value_range = MAX_VALUES[field_name] - MIN_VALUES[field_name]
+        if self.step and self.step > value_range:
+            raise ValueError('the step value ({}) is higher than the total range of the '
+                             'expression ({})'.format(self.step, value_range))
+
     def get_next_value(self, date, field):
         start = field.get_value(date)
         minval = field.get_min(date)
@@ -37,6 +44,9 @@ class AllExpression(object):
         if next <= maxval:
             return next
 
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.step == other.step
+
     def __str__(self):
         if self.step:
             return '*/%d' % self.step
@@ -51,7 +61,7 @@ class RangeExpression(AllExpression):
         r'(?P<first>\d+)(?:-(?P<last>\d+))?(?:/(?P<step>\d+))?$')
 
     def __init__(self, first, last=None, step=None):
-        AllExpression.__init__(self, step)
+        super(RangeExpression, self).__init__(step)
         first = asint(first)
         last = asint(last)
         if last is None and step is None:
@@ -61,25 +71,41 @@ class RangeExpression(AllExpression):
         self.first = first
         self.last = last
 
+    def validate_range(self, field_name):
+        from apscheduler.triggers.cron.fields import MIN_VALUES, MAX_VALUES
+
+        super(RangeExpression, self).validate_range(field_name)
+        if self.first < MIN_VALUES[field_name]:
+            raise ValueError('the first value ({}) is lower than the minimum value ({})'
+                             .format(self.first, MIN_VALUES[field_name]))
+        if self.last is not None and self.last > MAX_VALUES[field_name]:
+            raise ValueError('the last value ({}) is higher than the maximum value ({})'
+                             .format(self.last, MAX_VALUES[field_name]))
+        value_range = (self.last or MAX_VALUES[field_name]) - self.first
+        if self.step and self.step > value_range:
+            raise ValueError('the step value ({}) is higher than the total range of the '
+                             'expression ({})'.format(self.step, value_range))
+
     def get_next_value(self, date, field):
-        start = field.get_value(date)
+        startval = field.get_value(date)
         minval = field.get_min(date)
         maxval = field.get_max(date)
 
         # Apply range limits
         minval = max(minval, self.first)
-        if self.last is not None:
-            maxval = min(maxval, self.last)
-        start = max(start, minval)
+        maxval = min(maxval, self.last) if self.last is not None else maxval
+        nextval = max(minval, startval)
 
-        if not self.step:
-            next = start
-        else:
-            distance_to_next = (self.step - (start - minval)) % self.step
-            next = start + distance_to_next
+        # Apply the step if defined
+        if self.step:
+            distance_to_next = (self.step - (nextval - minval)) % self.step
+            nextval += distance_to_next
 
-        if next <= maxval:
-            return next
+        return nextval if nextval <= maxval else None
+
+    def __eq__(self, other):
+        return (isinstance(other, self.__class__) and self.first == other.first and
+                self.last == other.last)
 
     def __str__(self):
         if self.last != self.first and self.last is not None:
@@ -100,6 +126,37 @@ class RangeExpression(AllExpression):
         return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
 
 
+class MonthRangeExpression(RangeExpression):
+    value_re = re.compile(r'(?P<first>[a-z]+)(?:-(?P<last>[a-z]+))?', re.IGNORECASE)
+
+    def __init__(self, first, last=None):
+        try:
+            first_num = MONTHS.index(first.lower()) + 1
+        except ValueError:
+            raise ValueError('Invalid month name "%s"' % first)
+
+        if last:
+            try:
+                last_num = MONTHS.index(last.lower()) + 1
+            except ValueError:
+                raise ValueError('Invalid month name "%s"' % last)
+        else:
+            last_num = None
+
+        super(MonthRangeExpression, self).__init__(first_num, last_num)
+
+    def __str__(self):
+        if self.last != self.first and self.last is not None:
+            return '%s-%s' % (MONTHS[self.first - 1], MONTHS[self.last - 1])
+        return MONTHS[self.first - 1]
+
+    def __repr__(self):
+        args = ["'%s'" % MONTHS[self.first]]
+        if self.last != self.first and self.last is not None:
+            args.append("'%s'" % MONTHS[self.last - 1])
+        return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
+
+
 class WeekdayRangeExpression(RangeExpression):
     value_re = re.compile(r'(?P<first>[a-z]+)(?:-(?P<last>[a-z]+))?', re.IGNORECASE)
 
@@ -117,7 +174,7 @@ class WeekdayRangeExpression(RangeExpression):
         else:
             last_num = None
 
-        RangeExpression.__init__(self, first_num, last_num)
+        super(WeekdayRangeExpression, self).__init__(first_num, last_num)
 
     def __str__(self):
         if self.last != self.first and self.last is not None:
@@ -133,9 +190,11 @@ class WeekdayRangeExpression(RangeExpression):
 
 class WeekdayPositionExpression(AllExpression):
     options = ['1st', '2nd', '3rd', '4th', '5th', 'last']
-    value_re = re.compile(r'(?P<option_name>%s) +(?P<weekday_name>(?:\d+|\w+))' % '|'.join(options), re.IGNORECASE)
+    value_re = re.compile(r'(?P<option_name>%s) +(?P<weekday_name>(?:\d+|\w+))' %
+                          '|'.join(options), re.IGNORECASE)
 
     def __init__(self, option_name, weekday_name):
+        super(WeekdayPositionExpression, self).__init__(None)
         try:
             self.option_num = self.options.index(option_name.lower())
         except ValueError:
@@ -147,8 +206,7 @@ class WeekdayPositionExpression(AllExpression):
             raise ValueError('Invalid weekday name "%s"' % weekday_name)
 
     def get_next_value(self, date, field):
-        # Figure out the weekday of the month's first day and the number
-        # of days in that month
+        # Figure out the weekday of the month's first day and the number of days in that month
         first_day_wday, last_day = monthrange(date.year, date.month)
 
         # Calculate which day of the month is the first of the target weekdays
@@ -160,23 +218,28 @@ class WeekdayPositionExpression(AllExpression):
         if self.option_num < 5:
             target_day = first_hit_day + self.option_num * 7
         else:
-            target_day = first_hit_day + ((last_day - first_hit_day) / 7) * 7
+            target_day = first_hit_day + ((last_day - first_hit_day) // 7) * 7
 
         if target_day <= last_day and target_day >= date.day:
             return target_day
 
+    def __eq__(self, other):
+        return (super(WeekdayPositionExpression, self).__eq__(other) and
+                self.option_num == other.option_num and self.weekday == other.weekday)
+
     def __str__(self):
         return '%s %s' % (self.options[self.option_num], WEEKDAYS[self.weekday])
 
     def __repr__(self):
-        return "%s('%s', '%s')" % (self.__class__.__name__, self.options[self.option_num], WEEKDAYS[self.weekday])
+        return "%s('%s', '%s')" % (self.__class__.__name__, self.options[self.option_num],
+                                   WEEKDAYS[self.weekday])
 
 
 class LastDayOfMonthExpression(AllExpression):
     value_re = re.compile(r'last', re.IGNORECASE)
 
     def __init__(self):
-        pass
+        super(LastDayOfMonthExpression, self).__init__(None)
 
     def get_next_value(self, date, field):
         return monthrange(date.year, date.month)[1]

lib/apscheduler/triggers/cron/fields.py

@@ -1,22 +1,26 @@
-"""
-Fields represent CronTrigger options which map to :class:`~datetime.datetime`
-fields.
-"""
+"""Fields represent CronTrigger options which map to :class:`~datetime.datetime` fields."""
 
 from calendar import monthrange
+import re
+
+import six
 
 from apscheduler.triggers.cron.expressions import (
-    AllExpression, RangeExpression, WeekdayPositionExpression, LastDayOfMonthExpression, WeekdayRangeExpression)
+    AllExpression, RangeExpression, WeekdayPositionExpression, LastDayOfMonthExpression,
+    WeekdayRangeExpression, MonthRangeExpression)
 
 
-__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', 'WeekField', 'DayOfMonthField', 'DayOfWeekField')
+__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', 'WeekField',
+           'DayOfMonthField', 'DayOfWeekField')
 
 
-MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, 'day_of_week': 0, 'hour': 0, 'minute': 0, 'second': 0}
-MAX_VALUES = {'year': 2 ** 63, 'month': 12, 'day:': 31, 'week': 53, 'day_of_week': 6, 'hour': 23, 'minute': 59,
-              'second': 59}
-DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', 'day_of_week': '*', 'hour': 0, 'minute': 0,
-                  'second': 0}
+MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, 'day_of_week': 0, 'hour': 0,
+              'minute': 0, 'second': 0}
+MAX_VALUES = {'year': 9999, 'month': 12, 'day': 31, 'week': 53, 'day_of_week': 6, 'hour': 23,
+              'minute': 59, 'second': 59}
+DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', 'day_of_week': '*', 'hour': 0,
+                  'minute': 0, 'second': 0}
+SEPARATOR = re.compile(' *, *')
 
 
 class BaseField(object):
@@ -50,23 +54,29 @@ class BaseField(object):
         self.expressions = []
 
         # Split a comma-separated expression list, if any
-        exprs = str(exprs).strip()
-        if ',' in exprs:
-            for expr in exprs.split(','):
+        for expr in SEPARATOR.split(str(exprs).strip()):
             self.compile_expression(expr)
-        else:
-            self.compile_expression(exprs)
 
     def compile_expression(self, expr):
         for compiler in self.COMPILERS:
             match = compiler.value_re.match(expr)
             if match:
                 compiled_expr = compiler(**match.groupdict())
+
+                try:
+                    compiled_expr.validate_range(self.name)
+                except ValueError as e:
+                    exc = ValueError('Error validating expression {!r}: {}'.format(expr, e))
+                    six.raise_from(exc, None)
+
                 self.expressions.append(compiled_expr)
                 return
 
         raise ValueError('Unrecognized expression "%s" for field "%s"' % (expr, self.name))
 
+    def __eq__(self, other):
+        return isinstance(self, self.__class__) and self.expressions == other.expressions
+
     def __str__(self):
         expr_strings = (str(e) for e in self.expressions)
         return ','.join(expr_strings)
@@ -94,4 +104,8 @@ class DayOfWeekField(BaseField):
     COMPILERS = BaseField.COMPILERS + [WeekdayRangeExpression]
 
     def get_value(self, dateval):
-        return dateval.weekday()
+        return dateval.isoweekday() % 7
+
+
+class MonthField(BaseField):
+    COMPILERS = BaseField.COMPILERS + [MonthRangeExpression]

lib/apscheduler/triggers/date.py

@@ -14,15 +14,36 @@ class DateTrigger(BaseTrigger):
     :param datetime.tzinfo|str timezone: time zone for ``run_date`` if it doesn't have one already
     """
 
-    __slots__ = 'timezone', 'run_date'
+    __slots__ = 'run_date'
 
     def __init__(self, run_date=None, timezone=None):
         timezone = astimezone(timezone) or get_localzone()
-        self.run_date = convert_to_datetime(run_date or datetime.now(), timezone, 'run_date')
+        if run_date is not None:
+            self.run_date = convert_to_datetime(run_date, timezone, 'run_date')
+        else:
+            self.run_date = datetime.now(timezone)
 
     def get_next_fire_time(self, previous_fire_time, now):
         return self.run_date if previous_fire_time is None else None
 
+    def __getstate__(self):
+        return {
+            'version': 1,
+            'run_date': self.run_date
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 1:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only version 1 can be handled' %
+                (state['version'], self.__class__.__name__))
+
+        self.run_date = state['run_date']
+
     def __str__(self):
         return 'date[%s]' % datetime_repr(self.run_date)
 

lib/apscheduler/triggers/interval.py

@@ -9,8 +9,8 @@ from apscheduler.util import convert_to_datetime, timedelta_seconds, datetime_re
 
 class IntervalTrigger(BaseTrigger):
     """
-    Triggers on specified intervals, starting on ``start_date`` if specified, ``datetime.now()`` + interval
-    otherwise.
+    Triggers on specified intervals, starting on ``start_date`` if specified, ``datetime.now()`` +
+    interval otherwise.
 
     :param int weeks: number of weeks to wait
     :param int days: number of days to wait
@@ -20,12 +20,15 @@ class IntervalTrigger(BaseTrigger):
     :param datetime|str start_date: starting point for the interval calculation
     :param datetime|str end_date: latest possible date/time to trigger on
     :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations
+    :param int|None jitter: advance or delay the job execution by ``jitter`` seconds at most.
     """
 
-    __slots__ = 'timezone', 'start_date', 'end_date', 'interval'
+    __slots__ = 'timezone', 'start_date', 'end_date', 'interval', 'interval_length', 'jitter'
 
-    def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None, end_date=None, timezone=None):
-        self.interval = timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds)
+    def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None,
+                 end_date=None, timezone=None, jitter=None):
+        self.interval = timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes,
+                                  seconds=seconds)
         self.interval_length = timedelta_seconds(self.interval)
         if self.interval_length == 0:
             self.interval = timedelta(seconds=1)
@@ -33,9 +36,9 @@ class IntervalTrigger(BaseTrigger):
 
         if timezone:
             self.timezone = astimezone(timezone)
-        elif start_date and start_date.tzinfo:
+        elif isinstance(start_date, datetime) and start_date.tzinfo:
             self.timezone = start_date.tzinfo
-        elif end_date and end_date.tzinfo:
+        elif isinstance(end_date, datetime) and end_date.tzinfo:
             self.timezone = end_date.tzinfo
         else:
             self.timezone = get_localzone()
@@ -44,6 +47,8 @@ class IntervalTrigger(BaseTrigger):
         self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date')
         self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date')
 
+        self.jitter = jitter
+
     def get_next_fire_time(self, previous_fire_time, now):
         if previous_fire_time:
             next_fire_time = previous_fire_time + self.interval
@@ -54,12 +59,48 @@ class IntervalTrigger(BaseTrigger):
             next_interval_num = int(ceil(timediff_seconds / self.interval_length))
             next_fire_time = self.start_date + self.interval * next_interval_num
 
+        if self.jitter is not None:
+            next_fire_time = self._apply_jitter(next_fire_time, self.jitter, now)
+
         if not self.end_date or next_fire_time <= self.end_date:
             return self.timezone.normalize(next_fire_time)
 
+    def __getstate__(self):
+        return {
+            'version': 2,
+            'timezone': self.timezone,
+            'start_date': self.start_date,
+            'end_date': self.end_date,
+            'interval': self.interval,
+            'jitter': self.jitter,
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 2:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 2 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.timezone = state['timezone']
+        self.start_date = state['start_date']
+        self.end_date = state['end_date']
+        self.interval = state['interval']
+        self.interval_length = timedelta_seconds(self.interval)
+        self.jitter = state.get('jitter')
+
     def __str__(self):
         return 'interval[%s]' % str(self.interval)
 
     def __repr__(self):
-        return "<%s (interval=%r, start_date='%s')>" % (self.__class__.__name__, self.interval,
-                                                        datetime_repr(self.start_date))
+        options = ['interval=%r' % self.interval, 'start_date=%r' % datetime_repr(self.start_date)]
+        if self.end_date:
+            options.append("end_date=%r" % datetime_repr(self.end_date))
+        if self.jitter:
+            options.append('jitter=%s' % self.jitter)
+
+        return "<%s (%s, timezone='%s')>" % (
+            self.__class__.__name__, ', '.join(options), self.timezone)

lib/apscheduler/util.py

@@ -2,9 +2,9 @@
 
 from __future__ import division
 from datetime import date, datetime, time, timedelta, tzinfo
-from inspect import isfunction, ismethod, getargspec
 from calendar import timegm
 import re
+from functools import partial
 
 from pytz import timezone, utc
 import six
@@ -12,14 +12,16 @@ import six
 try:
     from inspect import signature
 except ImportError:  # pragma: nocover
-    try:
     from funcsigs import signature
+
+try:
+    from threading import TIMEOUT_MAX
 except ImportError:
-        signature = None
+    TIMEOUT_MAX = 4294967  # Maximum value accepted by Event.wait() on Windows
 
 __all__ = ('asint', 'asbool', 'astimezone', 'convert_to_datetime', 'datetime_to_utc_timestamp',
-           'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name', 'obj_to_ref',
-           'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args')
+           'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name',
+           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args')
 
 
 class _Undefined(object):
@@ -32,17 +34,18 @@ class _Undefined(object):
     def __repr__(self):
         return '<undefined>'
 
+
 undefined = _Undefined()  #: a unique object that only signifies that no value is defined
 
 
 def asint(text):
     """
-    Safely converts a string to an integer, returning None if the string is None.
+    Safely converts a string to an integer, returning ``None`` if the string is ``None``.
 
     :type text: str
     :rtype: int
-    """
 
+    """
     if text is not None:
         return int(text)
 
@@ -52,8 +55,8 @@ def asbool(obj):
     Interprets an object as a boolean value.
 
     :rtype: bool
-    """
 
+    """
     if isinstance(obj, str):
         obj = obj.strip().lower()
         if obj in ('true', 'yes', 'on', 'y', 't', '1'):
@@ -69,15 +72,19 @@ def astimezone(obj):
     Interprets an object as a timezone.
 
     :rtype: tzinfo
-    """
 
+    """
     if isinstance(obj, six.string_types):
         return timezone(obj)
     if isinstance(obj, tzinfo):
         if not hasattr(obj, 'localize') or not hasattr(obj, 'normalize'):
             raise TypeError('Only timezones from the pytz library are supported')
         if obj.zone == 'local':
-            raise ValueError('Unable to determine the name of the local timezone -- use an explicit timezone instead')
+            raise ValueError(
+                'Unable to determine the name of the local timezone -- you must explicitly '
+                'specify the name of the local timezone. Please refrain from using timezones like '
+                'EST to prevent problems with daylight saving time. Instead, use a locale based '
+                'timezone name (such as Europe/Helsinki).')
         return obj
     if obj is not None:
         raise TypeError('Expected tzinfo, got %s instead' % obj.__class__.__name__)
@@ -92,20 +99,20 @@ _DATE_REGEX = re.compile(
 def convert_to_datetime(input, tz, arg_name):
     """
     Converts the given object to a timezone aware datetime object.
+
     If a timezone aware datetime object is passed, it is returned unmodified.
     If a native datetime object is passed, it is given the specified timezone.
     If the input is a string, it is parsed as a datetime with the given timezone.
 
-    Date strings are accepted in three different forms: date only (Y-m-d),
-    date with time (Y-m-d H:M:S) or with date+time with microseconds
-    (Y-m-d H:M:S.micro).
+    Date strings are accepted in three different forms: date only (Y-m-d), date with time
+    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro).
 
     :param str|datetime input: the datetime or string to convert to a timezone aware datetime
     :param datetime.tzinfo tz: timezone to interpret ``input`` in
     :param str arg_name: the name of the argument (used in an error message)
     :rtype: datetime
-    """
 
+    """
     if input is None:
         return
     elif isinstance(input, datetime):
@@ -125,14 +132,16 @@ def convert_to_datetime(input, tz, arg_name):
     if datetime_.tzinfo is not None:
         return datetime_
     if tz is None:
-        raise ValueError('The "tz" argument must be specified if %s has no timezone information' % arg_name)
+        raise ValueError(
+            'The "tz" argument must be specified if %s has no timezone information' % arg_name)
     if isinstance(tz, six.string_types):
         tz = timezone(tz)
 
     try:
         return tz.localize(datetime_, is_dst=None)
     except AttributeError:
-        raise TypeError('Only pytz timezones are supported (need the localize() and normalize() methods)')
+        raise TypeError(
+            'Only pytz timezones are supported (need the localize() and normalize() methods)')
 
 
 def datetime_to_utc_timestamp(timeval):
@@ -141,8 +150,8 @@ def datetime_to_utc_timestamp(timeval):
 
     :type timeval: datetime
     :rtype: float
-    """
 
+    """
     if timeval is not None:
         return timegm(timeval.utctimetuple()) + timeval.microsecond / 1000000
 
@@ -153,8 +162,8 @@ def utc_timestamp_to_datetime(timestamp):
 
     :type timestamp: float
     :rtype: datetime
-    """
 
+    """
     if timestamp is not None:
         return datetime.fromtimestamp(timestamp, utc)
 
@@ -165,8 +174,8 @@ def timedelta_seconds(delta):
 
     :type delta: timedelta
     :rtype: float
-    """
 
+    """
     return delta.days * 24 * 60 * 60 + delta.seconds + \
         delta.microseconds / 1000000.0
 
@@ -176,8 +185,8 @@ def datetime_ceil(dateval):
     Rounds the given datetime object upwards.
 
     :type dateval: datetime
-    """
 
+    """
     if dateval.microsecond > 0:
         return dateval + timedelta(seconds=1, microseconds=-dateval.microsecond)
     return dateval
@@ -192,8 +201,8 @@ def get_callable_name(func):
     Returns the best available display name for the given function/callable.
 
     :rtype: str
-    """
 
+    """
     # the easy case (on Python 3.3+)
     if hasattr(func, '__qualname__'):
         return func.__qualname__
@@ -222,20 +231,24 @@ def get_callable_name(func):
 
 def obj_to_ref(obj):
     """
-    Returns the path to the given object.
+    Returns the path to the given callable.
 
     :rtype: str
+    :raises TypeError: if the given object is not callable
+    :raises ValueError: if the given object is a :class:`~functools.partial`, lambda or a nested
+        function
+
     """
+    if isinstance(obj, partial):
+        raise ValueError('Cannot create a reference to a partial()')
 
-    try:
-        ref = '%s:%s' % (obj.__module__, get_callable_name(obj))
-        obj2 = ref_to_obj(ref)
-        if obj != obj2:
-            raise ValueError
-    except Exception:
-        raise ValueError('Cannot determine the reference to %r' % obj)
+    name = get_callable_name(obj)
+    if '<lambda>' in name:
+        raise ValueError('Cannot create a reference to a lambda')
+    if '<locals>' in name:
+        raise ValueError('Cannot create a reference to a nested function')
 
-    return ref
+    return '%s:%s' % (obj.__module__, name)
 
 
 def ref_to_obj(ref):
@@ -243,8 +256,8 @@ def ref_to_obj(ref):
     Returns the object pointed to by ``ref``.
 
     :type ref: str
-    """
 
+    """
     if not isinstance(ref, six.string_types):
         raise TypeError('References must be strings')
     if ':' not in ref:
@@ -252,12 +265,12 @@ def ref_to_obj(ref):
 
     modulename, rest = ref.split(':', 1)
     try:
-        obj = __import__(modulename)
+        obj = __import__(modulename, fromlist=[rest])
     except ImportError:
         raise LookupError('Error resolving reference %s: could not import module' % ref)
 
     try:
-        for name in modulename.split('.')[1:] + rest.split('.'):
+        for name in rest.split('.'):
             obj = getattr(obj, name)
         return obj
     except Exception:
@@ -268,8 +281,8 @@ def maybe_ref(ref):
     """
     Returns the object that the given reference points to, if it is indeed a reference.
     If it is not a reference, the object is returned as-is.
-    """
 
+    """
     if not isinstance(ref, str):
         return ref
     return ref_to_obj(ref)
@@ -281,7 +294,8 @@ if six.PY2:
             return string.encode('ascii', 'backslashreplace')
         return string
 else:
-    repr_escape = lambda string: string
+    def repr_escape(string):
+        return string
 
 
 def check_callable_args(func, args, kwargs):
@@ -290,21 +304,23 @@ def check_callable_args(func, args, kwargs):
 
     :type args: tuple
     :type kwargs: dict
-    """
 
+    """
     pos_kwargs_conflicts = []  # parameters that have a match in both args and kwargs
     positional_only_kwargs = []  # positional-only parameters that have a match in kwargs
     unsatisfied_args = []  # parameters in signature that don't have a match in args or kwargs
     unsatisfied_kwargs = []  # keyword-only arguments that don't have a match in kwargs
     unmatched_args = list(args)  # args that didn't match any of the parameters in the signature
-    unmatched_kwargs = list(kwargs)  # kwargs that didn't match any of the parameters in the signature
-    has_varargs = has_var_kwargs = False  # indicates if the signature defines *args and **kwargs respectively
+    # kwargs that didn't match any of the parameters in the signature
+    unmatched_kwargs = list(kwargs)
+    # indicates if the signature defines *args and **kwargs respectively
+    has_varargs = has_var_kwargs = False
 
-    if signature:
     try:
         sig = signature(func)
     except ValueError:
-            return  # signature() doesn't work against every kind of callable
+        # signature() doesn't work against every kind of callable
+        return
 
     for param in six.itervalues(sig.parameters):
         if param.kind == param.POSITIONAL_OR_KEYWORD:
@@ -333,27 +349,6 @@ def check_callable_args(func, args, kwargs):
             has_varargs = True
         elif param.kind == param.VAR_KEYWORD:
             has_var_kwargs = True
-    else:
-        if not isfunction(func) and not ismethod(func) and hasattr(func, '__call__'):
-            func = func.__call__
-
-        try:
-            argspec = getargspec(func)
-        except TypeError:
-            return  # getargspec() doesn't work certain callables
-
-        argspec_args = argspec.args if not ismethod(func) else argspec.args[1:]
-        has_varargs = bool(argspec.varargs)
-        has_var_kwargs = bool(argspec.keywords)
-        for arg, default in six.moves.zip_longest(argspec_args, argspec.defaults or (), fillvalue=undefined):
-            if arg in unmatched_kwargs and unmatched_args:
-                pos_kwargs_conflicts.append(arg)
-            elif unmatched_args:
-                del unmatched_args[0]
-            elif arg in unmatched_kwargs:
-                unmatched_kwargs.remove(arg)
-            elif default is undefined:
-                unsatisfied_args.append(arg)
 
     # Make sure there are no conflicts between args and kwargs
     if pos_kwargs_conflicts:
@@ -365,21 +360,26 @@ def check_callable_args(func, args, kwargs):
         raise ValueError('The following arguments cannot be given as keyword arguments: %s' %
                          ', '.join(positional_only_kwargs))
 
-    # Check that the number of positional arguments minus the number of matched kwargs matches the argspec
+    # Check that the number of positional arguments minus the number of matched kwargs matches the
+    # argspec
     if unsatisfied_args:
-        raise ValueError('The following arguments have not been supplied: %s' % ', '.join(unsatisfied_args))
+        raise ValueError('The following arguments have not been supplied: %s' %
+                         ', '.join(unsatisfied_args))
 
     # Check that all keyword-only arguments have been supplied
     if unsatisfied_kwargs:
-        raise ValueError('The following keyword-only arguments have not been supplied in kwargs: %s' %
+        raise ValueError(
+            'The following keyword-only arguments have not been supplied in kwargs: %s' %
             ', '.join(unsatisfied_kwargs))
 
     # Check that the callable can accept the given number of positional arguments
     if not has_varargs and unmatched_args:
-        raise ValueError('The list of positional arguments is longer than the target callable can handle '
+        raise ValueError(
+            'The list of positional arguments is longer than the target callable can handle '
             '(allowed: %d, given in args: %d)' % (len(args) - len(unmatched_args), len(args)))
 
     # Check that the callable can accept the given keyword arguments
     if not has_var_kwargs and unmatched_kwargs:
-        raise ValueError('The target callable does not accept the following keyword arguments: %s' %
+        raise ValueError(
+            'The target callable does not accept the following keyword arguments: %s' %
             ', '.join(unmatched_kwargs))

lib/arrow/__init__.py

@@ -4,5 +4,5 @@ from .arrow import Arrow
 from .factory import ArrowFactory
 from .api import get, now, utcnow
 
-__version__ = '0.7.0'
+__version__ = '0.10.0'
 VERSION = __version__

lib/arrow/api.py

@@ -51,5 +51,5 @@ def factory(type):
     return ArrowFactory(type)
 
 
-__all__ = ['get', 'utcnow', 'now', 'factory', 'iso']
+__all__ = ['get', 'utcnow', 'now', 'factory']
 

lib/arrow/arrow.py

@@ -12,6 +12,8 @@ from dateutil import tz as dateutil_tz
 from dateutil.relativedelta import relativedelta
 import calendar
 import sys
+import warnings
+
 
 from arrow import util, locales, parser, formatter
 
@@ -45,6 +47,7 @@ class Arrow(object):
 
     _ATTRS = ['year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond']
     _ATTRS_PLURAL = ['{0}s'.format(a) for a in _ATTRS]
+    _MONTHS_PER_QUARTER = 3
 
     def __init__(self, year, month, day, hour=0, minute=0, second=0, microsecond=0,
                  tzinfo=None):
@@ -306,6 +309,9 @@ class Arrow(object):
         if name == 'week':
             return self.isocalendar()[1]
 
+        if name == 'quarter':
+            return int((self.month-1)/self._MONTHS_PER_QUARTER) + 1
+
         if not name.startswith('_'):
             value = getattr(self._datetime, name, None)
 
@@ -378,16 +384,16 @@ class Arrow(object):
         >>> arw.replace(year=2014, month=6)
         <Arrow [2014-06-11T22:27:34.787885+00:00]>
 
-        Use plural property names to shift their current value relatively:
-
-        >>> arw.replace(years=1, months=-1)
-        <Arrow [2014-04-11T22:27:34.787885+00:00]>
-
         You can also provide a timezone expression can also be replaced:
 
         >>> arw.replace(tzinfo=tz.tzlocal())
         <Arrow [2013-05-11T22:27:34.787885-07:00]>
 
+       Use plural property names to shift their current value relatively (**deprecated**):
+
+       >>> arw.replace(years=1, months=-1)
+       <Arrow [2014-04-11T22:27:34.787885+00:00]>
+
         Recognized timezone expressions:
 
             - A ``tzinfo`` object.
@@ -398,21 +404,29 @@ class Arrow(object):
         '''
 
         absolute_kwargs = {}
-        relative_kwargs = {}
+        relative_kwargs = {}  # TODO: DEPRECATED; remove in next release
 
         for key, value in kwargs.items():
 
             if key in self._ATTRS:
                 absolute_kwargs[key] = value
-            elif key in self._ATTRS_PLURAL or key == 'weeks':
+            elif key in self._ATTRS_PLURAL or key in ['weeks', 'quarters']:
+                # TODO: DEPRECATED
+                warnings.warn("replace() with plural property to shift value"
+                              "is deprecated, use shift() instead",
+                              DeprecationWarning)
                 relative_kwargs[key] = value
-            elif key == 'week':
-                raise AttributeError('setting absolute week is not supported')
+            elif key in ['week', 'quarter']:
+                raise AttributeError('setting absolute {0} is not supported'.format(key))
             elif key !='tzinfo':
-                raise AttributeError()
+                raise AttributeError('unknown attribute: "{0}"'.format(key))
+
+        # core datetime does not support quarters, translate to months.
+        relative_kwargs.setdefault('months', 0)
+        relative_kwargs['months'] += relative_kwargs.pop('quarters', 0) * self._MONTHS_PER_QUARTER
 
         current = self._datetime.replace(**absolute_kwargs)
-        current += relativedelta(**relative_kwargs)
+        current += relativedelta(**relative_kwargs) # TODO: DEPRECATED
 
         tzinfo = kwargs.get('tzinfo')
 
@@ -422,9 +436,41 @@ class Arrow(object):
 
         return self.fromdatetime(current)
 
+    def shift(self, **kwargs):
+        ''' Returns a new :class:`Arrow <arrow.arrow.Arrow>` object with attributes updated
+        according to inputs.
+
+        Use plural property names to shift their current value relatively:
+
+        >>> import arrow
+        >>> arw = arrow.utcnow()
+        >>> arw
+        <Arrow [2013-05-11T22:27:34.787885+00:00]>
+        >>> arw.shift(years=1, months=-1)
+        <Arrow [2014-04-11T22:27:34.787885+00:00]>
+
+        '''
+
+        relative_kwargs = {}
+
+        for key, value in kwargs.items():
+
+            if key in self._ATTRS_PLURAL or key in ['weeks', 'quarters']:
+                relative_kwargs[key] = value
+            else:
+                raise AttributeError()
+
+        # core datetime does not support quarters, translate to months.
+        relative_kwargs.setdefault('months', 0)
+        relative_kwargs['months'] += relative_kwargs.pop('quarters', 0) * self._MONTHS_PER_QUARTER
+
+        current = self._datetime + relativedelta(**relative_kwargs)
+
+        return self.fromdatetime(current)
+
     def to(self, tz):
-        ''' Returns a new :class:`Arrow <arrow.arrow.Arrow>` object, converted to the target
-        timezone.
+        ''' Returns a new :class:`Arrow <arrow.arrow.Arrow>` object, converted
+        to the target timezone.
 
         :param tz: an expression representing a timezone.
 
@@ -587,6 +633,7 @@ class Arrow(object):
             Defaults to now in the current :class:`Arrow <arrow.arrow.Arrow>` object's timezone.
         :param locale: (optional) a ``str`` specifying a locale.  Defaults to 'en_us'.
         :param only_distance: (optional) returns only time difference eg: "11 seconds" without "in" or "ago" part.
+
         Usage::
 
             >>> earlier = arrow.utcnow().replace(hours=-2)
@@ -651,7 +698,8 @@ class Arrow(object):
         elif diff < 29808000:
             self_months = self._datetime.year * 12 + self._datetime.month
             other_months = dt.year * 12 + dt.month
-            months = sign * abs(other_months - self_months)
+
+            months = sign * int(max(abs(other_months - self_months), 2))
 
             return locale.describe('months', months, only_distance=only_distance)
 
@@ -676,7 +724,7 @@ class Arrow(object):
 
     def __sub__(self, other):
 
-        if isinstance(other, timedelta):
+        if isinstance(other, (timedelta, relativedelta)):
             return self.fromdatetime(self._datetime - other, self._datetime.tzinfo)
 
         elif isinstance(other, datetime):
@@ -688,7 +736,11 @@ class Arrow(object):
         raise TypeError()
 
     def __rsub__(self, other):
-        return self.__sub__(other)
+
+        if isinstance(other, datetime):
+            return other - self._datetime
+
+        raise TypeError()
 
 
     # comparisons
@@ -702,8 +754,6 @@ class Arrow(object):
         if not isinstance(other, (Arrow, datetime)):
             return False
 
-        other = self._get_datetime(other)
-
         return self._datetime == self._get_datetime(other)
 
     def __ne__(self, other):
@@ -882,7 +932,9 @@ class Arrow(object):
             return cls.max, limit
 
         else:
+            if limit is None:
                 return end, sys.maxsize
+            return end, limit
 
     @staticmethod
     def _get_timestamp_from_input(timestamp):

lib/arrow/formatter.py

@@ -94,7 +94,7 @@ class DateTimeFormatter(object):
             tz = dateutil_tz.tzutc() if dt.tzinfo is None else dt.tzinfo
             total_minutes = int(util.total_seconds(tz.utcoffset(dt)) / 60)
 
-            sign = '+' if total_minutes > 0 else '-'
+            sign = '+' if total_minutes >= 0 else '-'
             total_minutes = abs(total_minutes)
             hour, minute = divmod(total_minutes, 60)
 

lib/arrow/locales.py

@@ -7,8 +7,8 @@ import sys
 
 
 def get_locale(name):
-    '''Returns an appropriate :class:`Locale <locale.Locale>` corresponding
-    to an inpute locale name.
+    '''Returns an appropriate :class:`Locale <arrow.locales.Locale>`
+    corresponding to an inpute locale name.
 
     :param name: the name of the locale.
 
@@ -186,7 +186,7 @@ class Locale(object):
 
 class EnglishLocale(Locale):
 
-    names = ['en', 'en_us', 'en_gb', 'en_au', 'en_be', 'en_jp', 'en_za']
+    names = ['en', 'en_us', 'en_gb', 'en_au', 'en_be', 'en_jp', 'en_za', 'en_ca']
 
     past = '{0} ago'
     future = 'in {0}'
@@ -263,10 +263,10 @@ class ItalianLocale(Locale):
     day_names = ['', 'lunedì', 'martedì', 'mercoledì', 'giovedì', 'venerdì', 'sabato', 'domenica']
     day_abbreviations = ['', 'lun', 'mar', 'mer', 'gio', 'ven', 'sab', 'dom']
 
-    ordinal_day_re = r'((?P<value>[1-3]?[0-9](?=°))°)'
+    ordinal_day_re = r'((?P<value>[1-3]?[0-9](?=[ºª]))[ºª])'
 
     def _ordinal_number(self, n):
-        return '{0}°'.format(n)
+        return '{0}º'.format(n)
 
 
 class SpanishLocale(Locale):
@@ -297,10 +297,10 @@ class SpanishLocale(Locale):
     day_names = ['', 'lunes', 'martes', 'miércoles', 'jueves', 'viernes', 'sábado', 'domingo']
     day_abbreviations = ['', 'lun', 'mar', 'mie', 'jue', 'vie', 'sab', 'dom']
 
-    ordinal_day_re = r'((?P<value>[1-3]?[0-9](?=°))°)'
+    ordinal_day_re = r'((?P<value>[1-3]?[0-9](?=[ºª]))[ºª])'
 
     def _ordinal_number(self, n):
-        return '{0}°'.format(n)
+        return '{0}º'.format(n)
 
 
 class FrenchLocale(Locale):
@@ -379,7 +379,7 @@ class JapaneseLocale(Locale):
 
     timeframes = {
         'now': '現在',
-        'seconds': '秒',
+        'seconds': '数秒',
         'minute': '1分',
         'minutes': '{0}分',
         'hour': '1時間',
@@ -560,7 +560,7 @@ class KoreanLocale(Locale):
     timeframes = {
         'now': '지금',
         'seconds': '몇 초',
-        'minute': '일 분',
+        'minute': '1분',
         'minutes': '{0}분',
         'hour': '1시간',
         'hours': '{0}시간',
@@ -1034,7 +1034,7 @@ class TurkishLocale(Locale):
         'days': '{0} gün',
         'month': 'bir ay',
         'months': '{0} ay',
-        'year': 'a yıl',
+        'year': 'yıl',
         'years': '{0} yıl',
     }
 
@@ -1047,6 +1047,37 @@ class TurkishLocale(Locale):
     day_abbreviations = ['', 'Pzt', 'Sal', 'Çar', 'Per', 'Cum', 'Cmt', 'Paz']
 
 
+class AzerbaijaniLocale(Locale):
+
+    names = ['az', 'az_az']
+
+    past = '{0} əvvəl'
+    future = '{0} sonra'
+
+    timeframes = {
+        'now': 'indi',
+        'seconds': 'saniyə',
+        'minute': 'bir dəqiqə',
+        'minutes': '{0} dəqiqə',
+        'hour': 'bir saat',
+        'hours': '{0} saat',
+        'day': 'bir gün',
+        'days': '{0} gün',
+        'month': 'bir ay',
+        'months': '{0} ay',
+        'year': 'il',
+        'years': '{0} il',
+    }
+
+    month_names = ['', 'Yanvar', 'Fevral', 'Mart', 'Aprel', 'May', 'İyun', 'İyul',
+        'Avqust', 'Sentyabr', 'Oktyabr', 'Noyabr', 'Dekabr']
+    month_abbreviations = ['', 'Yan', 'Fev', 'Mar', 'Apr', 'May', 'İyn', 'İyl', 'Avq',
+        'Sen', 'Okt', 'Noy', 'Dek']
+
+    day_names = ['', 'Bazar ertəsi', 'Çərşənbə axşamı', 'Çərşənbə', 'Cümə axşamı', 'Cümə', 'Şənbə', 'Bazar']
+    day_abbreviations = ['', 'Ber', 'Çax', 'Çər', 'Cax', 'Cüm', 'Şnb', 'Bzr']
+
+
 class ArabicLocale(Locale):
 
     names = ['ar', 'ar_eg']
@@ -1205,11 +1236,11 @@ class HindiLocale(Locale):
     future = '{0} बाद'
 
     timeframes = {
-        'now': 'अभि',
+        'now': 'अभी',
         'seconds': 'सेकंड्',
         'minute': 'एक मिनट ',
         'minutes': '{0} मिनट ',
-        'hour': 'एक घंट',
+        'hour': 'एक घंटा',
         'hours': '{0} घंटे',
         'day': 'एक दिन',
         'days': '{0} दिन',
@@ -1226,8 +1257,8 @@ class HindiLocale(Locale):
         'PM': 'शाम',
     }
 
-    month_names = ['', 'जनवरी', 'फ़रवरी', 'मार्च', 'अप्रैल ', 'मई', 'जून', 'जुलाई',
-                   'आगस्त', 'सितम्बर', 'अकतूबर', 'नवेम्बर', 'दिसम्बर']
+    month_names = ['', 'जनवरी', 'फरवरी', 'मार्च', 'अप्रैल ', 'मई', 'जून', 'जुलाई',
+                   'अगस्त', 'सितंबर', 'अक्टूबर', 'नवंबर', 'दिसंबर']
     month_abbreviations = ['', 'जन', 'फ़र', 'मार्च', 'अप्रै', 'मई', 'जून', 'जुलाई', 'आग',
                            'सित', 'अकत', 'नवे', 'दिस']
 
@@ -1284,7 +1315,8 @@ class CzechLocale(Locale):
 
 
     def _format_timeframe(self, timeframe, delta):
-        '''Czech aware time frame format function, takes into account the differences between past and future forms.'''
+        '''Czech aware time frame format function, takes into account
+        the differences between past and future forms.'''
         form = self.timeframes[timeframe]
         if isinstance(form, dict):
             if delta == 0:
@@ -1303,6 +1335,78 @@ class CzechLocale(Locale):
 
         return form.format(delta)
 
+
+class SlovakLocale(Locale):
+    names = ['sk', 'sk_sk']
+
+    timeframes = {
+        'now': 'Teraz',
+        'seconds': {
+            'past': 'pár sekundami',
+            'future': ['{0} sekundy', '{0} sekúnd']
+        },
+        'minute': {'past': 'minútou', 'future': 'minútu', 'zero': '{0} minút'},
+        'minutes': {
+            'past': '{0} minútami',
+            'future': ['{0} minúty', '{0} minút']
+        },
+        'hour': {'past': 'hodinou', 'future': 'hodinu', 'zero': '{0} hodín'},
+        'hours': {
+            'past': '{0} hodinami',
+            'future': ['{0} hodiny', '{0} hodín']
+        },
+        'day': {'past': 'dňom', 'future': 'deň', 'zero': '{0} dní'},
+        'days': {
+            'past': '{0} dňami',
+            'future': ['{0} dni', '{0} dní']
+        },
+        'month': {'past': 'mesiacom', 'future': 'mesiac', 'zero': '{0} mesiacov'},
+        'months': {
+            'past': '{0} mesiacmi',
+            'future': ['{0} mesiace', '{0} mesiacov']
+        },
+        'year': {'past': 'rokom', 'future': 'rok', 'zero': '{0} rokov'},
+        'years': {
+            'past': '{0} rokmi',
+            'future': ['{0} roky', '{0} rokov']
+        }
+    }
+
+    past = 'Pred {0}'
+    future = 'O {0}'
+
+    month_names = ['', 'január', 'február', 'marec', 'apríl', 'máj', 'jún',
+        'júl', 'august', 'september', 'október', 'november', 'december']
+    month_abbreviations = ['', 'jan', 'feb', 'mar', 'apr', 'máj', 'jún', 'júl',
+        'aug', 'sep', 'okt', 'nov', 'dec']
+
+    day_names = ['', 'pondelok', 'utorok', 'streda', 'štvrtok', 'piatok',
+        'sobota', 'nedeľa']
+    day_abbreviations = ['', 'po', 'ut', 'st', 'št', 'pi', 'so', 'ne']
+
+
+    def _format_timeframe(self, timeframe, delta):
+        '''Slovak aware time frame format function, takes into account
+        the differences between past and future forms.'''
+        form = self.timeframes[timeframe]
+        if isinstance(form, dict):
+            if delta == 0:
+                form = form['zero']  # And *never* use 0 in the singular!
+            elif delta > 0:
+                form = form['future']
+            else:
+                form = form['past']
+        delta = abs(delta)
+
+        if isinstance(form, list):
+            if 2 <= delta % 10 <= 4 and (delta % 100 < 10 or delta % 100 >= 20):
+                form = form[0]
+            else:
+                form = form[1]
+
+        return form.format(delta)
+
+
 class FarsiLocale(Locale):
 
     names = ['fa', 'fa_ir']
@@ -1475,10 +1579,10 @@ def _map_locales():
 
     return locales
 
-class CatalaLocale(Locale):
-    names = ['ca', 'ca_ca']
+class CatalanLocale(Locale):
+    names = ['ca', 'ca_es', 'ca_ad', 'ca_fr', 'ca_it']
     past = 'Fa {0}'
-    future = '{0}' # I don't know what's the right phrase in catala for the future.
+    future = 'En {0}'
 
     timeframes = {
         'now': 'Ara mateix',
@@ -1490,15 +1594,15 @@ class CatalaLocale(Locale):
         'day': 'un dia',
         'days': '{0} dies',
         'month': 'un mes',
-        'months': '{0} messos',
+        'months': '{0} mesos',
         'year': 'un any',
         'years': '{0} anys',
     }
 
-    month_names = ['', 'Jener', 'Febrer', 'Març', 'Abril', 'Maig', 'Juny', 'Juliol', 'Agost', 'Setembre', 'Octubre', 'Novembre', 'Decembre']
-    month_abbreviations = ['', 'Jener', 'Febrer', 'Març', 'Abril', 'Maig', 'Juny', 'Juliol', 'Agost', 'Setembre', 'Octubre', 'Novembre', 'Decembre']
-    day_names = ['', 'Dilluns', 'Dimars', 'Dimecres', 'Dijous', 'Divendres', 'Disabte', 'Diumenge']
-    day_abbreviations = ['', 'Dilluns', 'Dimars', 'Dimecres', 'Dijous', 'Divendres', 'Disabte', 'Diumenge']
+    month_names = ['', 'Gener', 'Febrer', 'Març', 'Abril', 'Maig', 'Juny', 'Juliol', 'Agost', 'Setembre', 'Octubre', 'Novembre', 'Desembre']
+    month_abbreviations = ['', 'Gener', 'Febrer', 'Març', 'Abril', 'Maig', 'Juny', 'Juliol', 'Agost', 'Setembre', 'Octubre', 'Novembre', 'Desembre']
+    day_names = ['', 'Dilluns', 'Dimarts', 'Dimecres', 'Dijous', 'Divendres', 'Dissabte', 'Diumenge']
+    day_abbreviations = ['', 'Dilluns', 'Dimarts', 'Dimecres', 'Dijous', 'Divendres', 'Dissabte', 'Diumenge']
 
 class BasqueLocale(Locale):
     names = ['eu', 'eu_eu']
@@ -1587,6 +1691,50 @@ class HungarianLocale(Locale):
         return form.format(abs(delta))
 
 
+class EsperantoLocale(Locale):
+    names = ['eo', 'eo_xx']
+    past = 'antaŭ {0}'
+    future = 'post {0}'
+
+    timeframes = {
+        'now': 'nun',
+        'seconds': 'kelkaj sekundoj',
+        'minute': 'unu minuto',
+        'minutes': '{0} minutoj',
+        'hour': 'un horo',
+        'hours': '{0} horoj',
+        'day': 'unu tago',
+        'days': '{0} tagoj',
+        'month': 'unu monato',
+        'months': '{0} monatoj',
+        'year': 'unu jaro',
+        'years': '{0} jaroj',
+    }
+
+    month_names = ['', 'januaro', 'februaro', 'marto', 'aprilo', 'majo',
+                   'junio', 'julio', 'aŭgusto', 'septembro', 'oktobro',
+                   'novembro', 'decembro']
+    month_abbreviations = ['', 'jan', 'feb', 'mar', 'apr', 'maj', 'jun',
+                           'jul', 'aŭg', 'sep', 'okt', 'nov', 'dec']
+
+    day_names = ['', 'lundo', 'mardo', 'merkredo', 'ĵaŭdo', 'vendredo',
+                 'sabato', 'dimanĉo']
+    day_abbreviations = ['', 'lun', 'mar', 'mer', 'ĵaŭ', 'ven',
+                         'sab', 'dim']
+
+    meridians = {
+        'am': 'atm',
+        'pm': 'ptm',
+        'AM': 'ATM',
+        'PM': 'PTM',
+    }
+
+    ordinal_day_re = r'((?P<value>[1-3]?[0-9](?=a))a)'
+
+    def _ordinal_number(self, n):
+        return '{0}a'.format(n)
+
+
 class ThaiLocale(Locale):
 
     names = ['th', 'th_th']
@@ -1700,4 +1848,164 @@ class BengaliLocale(Locale):
             return '{0}ষ্ঠ'.format(n)
 
 
+class RomanshLocale(Locale):
+
+    names = ['rm', 'rm_ch']
+
+    past = 'avant {0}'
+    future = 'en {0}'
+
+    timeframes = {
+        'now': 'en quest mument',
+        'seconds': 'secundas',
+        'minute': 'ina minuta',
+        'minutes': '{0} minutas',
+        'hour': 'in\'ura',
+        'hours': '{0} ura',
+        'day': 'in di',
+        'days': '{0} dis',
+        'month': 'in mais',
+        'months': '{0} mais',
+        'year': 'in onn',
+        'years': '{0} onns',
+    }
+
+    month_names = [
+        '', 'schaner', 'favrer', 'mars', 'avrigl', 'matg', 'zercladur',
+        'fanadur', 'avust', 'settember', 'october', 'november', 'december'
+    ]
+
+    month_abbreviations = [
+        '', 'schan', 'fav', 'mars', 'avr', 'matg', 'zer', 'fan', 'avu',
+        'set', 'oct', 'nov', 'dec'
+    ]
+
+    day_names = [
+        '', 'glindesdi', 'mardi', 'mesemna', 'gievgia', 'venderdi',
+        'sonda', 'dumengia'
+    ]
+
+    day_abbreviations = [
+        '', 'gli', 'ma', 'me', 'gie', 've', 'so', 'du'
+    ]
+
+
+class SwissLocale(Locale):
+
+    names = ['de', 'de_ch']
+
+    past = 'vor {0}'
+    future = 'in {0}'
+
+    timeframes = {
+            'now': 'gerade eben',
+            'seconds':  'Sekunden',
+            'minute': 'einer Minute',
+            'minutes': '{0} Minuten',
+            'hour': 'einer Stunde',
+            'hours': '{0} Stunden',
+            'day': 'einem Tag',
+            'days': '{0} Tage',
+            'month': 'einem Monat',
+            'months': '{0} Monaten',
+            'year': 'einem Jahr',
+            'years': '{0} Jahren',
+        }
+
+    month_names = [
+            '', 'Januar', 'Februar', 'März', 'April', 'Mai', 'Juni', 'Juli',
+            'August', 'September', 'Oktober', 'November', 'Dezember'
+        ]
+
+    month_abbreviations = [
+            '', 'Jan', 'Feb', 'Mär', 'Apr', 'Mai', 'Jun', 'Jul', 'Aug', 'Sep',
+            'Okt', 'Nov', 'Dez'
+        ]
+
+    day_names = [
+            '', 'Montag', 'Dienstag', 'Mittwoch', 'Donnerstag', 'Freitag',
+            'Samstag', 'Sonntag'
+        ]
+
+    day_abbreviations = [
+            '', 'Mo', 'Di', 'Mi', 'Do', 'Fr', 'Sa', 'So'
+        ]
+
+
+class RomanianLocale(Locale):
+    names = ['ro', 'ro_ro']
+
+    past = '{0} în urmă'
+    future = 'peste {0}'
+
+    timeframes = {
+        'now': 'acum',
+        'seconds': 'câteva secunde',
+        'minute': 'un minut',
+        'minutes': '{0} minute',
+        'hour': 'o oră',
+        'hours': '{0} ore',
+        'day': 'o zi',
+        'days': '{0} zile',
+        'month': 'o lună',
+        'months': '{0} luni',
+        'year': 'un an',
+        'years': '{0} ani',
+    }
+
+    month_names = ['', 'ianuarie', 'februarie', 'martie', 'aprilie', 'mai', 'iunie', 'iulie',
+                   'august', 'septembrie', 'octombrie', 'noiembrie', 'decembrie']
+    month_abbreviations = ['', 'ian', 'febr', 'mart', 'apr', 'mai', 'iun', 'iul', 'aug', 'sept', 'oct', 'nov', 'dec']
+
+    day_names = ['', 'luni', 'marți', 'miercuri', 'joi', 'vineri', 'sâmbătă', 'duminică']
+    day_abbreviations = ['', 'Lun', 'Mar', 'Mie', 'Joi', 'Vin', 'Sâm', 'Dum']
+
+
+class SlovenianLocale(Locale):
+    names = ['sl', 'sl_si']
+
+    past = 'pred {0}'
+    future = 'čez {0}'
+
+    timeframes = {
+        'now': 'zdaj',
+        'seconds': 'sekund',
+        'minute': 'minuta',
+        'minutes': '{0} minutami',
+        'hour': 'uro',
+        'hours': '{0} ur',
+        'day': 'dan',
+        'days': '{0} dni',
+        'month': 'mesec',
+        'months': '{0} mesecev',
+        'year': 'leto',
+        'years': '{0} let',
+    }
+
+    meridians = {
+        'am': '',
+        'pm': '',
+        'AM': '',
+        'PM': '',
+    }
+
+    month_names = [
+        '', 'Januar', 'Februar', 'Marec', 'April', 'Maj', 'Junij', 'Julij',
+        'Avgust', 'September', 'Oktober', 'November', 'December'
+    ]
+
+    month_abbreviations = [
+        '', 'Jan', 'Feb', 'Mar', 'Apr', 'Maj', 'Jun', 'Jul', 'Avg',
+        'Sep', 'Okt', 'Nov', 'Dec'
+    ]
+
+    day_names = [
+        '', 'Ponedeljek', 'Torek', 'Sreda', 'Četrtek', 'Petek', 'Sobota', 'Nedelja'
+    ]
+
+    day_abbreviations = [
+        '', 'Pon', 'Tor', 'Sre', 'Čet', 'Pet', 'Sob', 'Ned'
+    ]
+
+
 _locales = _map_locales()

lib/arrow/parser.py

@@ -5,7 +5,6 @@ from __future__ import unicode_literals
 from datetime import datetime
 from dateutil import tz
 import re
-
 from arrow import locales
 
 
@@ -15,16 +14,14 @@ class ParserError(RuntimeError):
 
 class DateTimeParser(object):
 
-    _FORMAT_RE = re.compile('(YYY?Y?|MM?M?M?|Do|DD?D?D?|HH?|hh?|mm?|ss?|SS?S?S?S?S?|ZZ?Z?|a|A|X)')
+    _FORMAT_RE = re.compile('(YYY?Y?|MM?M?M?|Do|DD?D?D?|d?d?d?d|HH?|hh?|mm?|ss?|S+|ZZ?Z?|a|A|X)')
+    _ESCAPE_RE = re.compile('\[[^\[\]]*\]')
 
-    _ONE_THROUGH_SIX_DIGIT_RE = re.compile('\d{1,6}')
-    _ONE_THROUGH_FIVE_DIGIT_RE = re.compile('\d{1,5}')
-    _ONE_THROUGH_FOUR_DIGIT_RE = re.compile('\d{1,4}')
-    _ONE_TWO_OR_THREE_DIGIT_RE = re.compile('\d{1,3}')
+    _ONE_OR_MORE_DIGIT_RE = re.compile('\d+')
     _ONE_OR_TWO_DIGIT_RE = re.compile('\d{1,2}')
     _FOUR_DIGIT_RE = re.compile('\d{4}')
     _TWO_DIGIT_RE = re.compile('\d{2}')
-    _TZ_RE = re.compile('[+\-]?\d{2}:?\d{2}')
+    _TZ_RE = re.compile('[+\-]?\d{2}:?(\d{2})?')
     _TZ_NAME_RE = re.compile('\w[\w+\-/]+')
 
 
@@ -47,12 +44,7 @@ class DateTimeParser(object):
         'ZZZ': _TZ_NAME_RE,
         'ZZ': _TZ_RE,
         'Z': _TZ_RE,
-        'SSSSSS': _ONE_THROUGH_SIX_DIGIT_RE,
-        'SSSSS': _ONE_THROUGH_FIVE_DIGIT_RE,
-        'SSSS': _ONE_THROUGH_FOUR_DIGIT_RE,
-        'SSS': _ONE_TWO_OR_THREE_DIGIT_RE,
-        'SS': _ONE_OR_TWO_DIGIT_RE,
-        'S': re.compile('\d'),
+        'S': _ONE_OR_MORE_DIGIT_RE,
     }
 
     MARKERS = ['YYYY', 'MM', 'DD']
@@ -67,6 +59,10 @@ class DateTimeParser(object):
             'MMM': self._choice_re(self.locale.month_abbreviations[1:],
                                    re.IGNORECASE),
             'Do': re.compile(self.locale.ordinal_day_re),
+            'dddd': self._choice_re(self.locale.day_names[1:], re.IGNORECASE),
+            'ddd': self._choice_re(self.locale.day_abbreviations[1:],
+                                   re.IGNORECASE),
+            'd' : re.compile("[1-7]"),
             'a': self._choice_re(
                 (self.locale.meridians['am'], self.locale.meridians['pm'])
             ),
@@ -88,11 +84,10 @@ class DateTimeParser(object):
             time_parts = re.split('[+-]', time_string, 1)
             has_tz = len(time_parts) > 1
             has_seconds = time_parts[0].count(':') > 1
-            has_subseconds = '.' in time_parts[0]
+            has_subseconds = re.search('[.,]', time_parts[0])
 
             if has_subseconds:
-                subseconds_token = 'S' * min(len(re.split('\D+', time_parts[0].split('.')[1], 1)[0]), 6)
-                formats = ['YYYY-MM-DDTHH:mm:ss.%s' % subseconds_token]
+                formats = ['YYYY-MM-DDTHH:mm:ss%sS' % has_subseconds.group()]
             elif has_seconds:
                 formats = ['YYYY-MM-DDTHH:mm:ss']
             else:
@@ -123,10 +118,18 @@ class DateTimeParser(object):
         # we construct a new string by replacing each
         # token by its pattern:
         # 'YYYY-MM-DD' -> '(?P<YYYY>\d{4})-(?P<MM>\d{2})-(?P<DD>\d{2})'
-        fmt_pattern = fmt
         tokens = []
         offset = 0
-        for m in self._FORMAT_RE.finditer(fmt):
+
+        # Extract the bracketed expressions to be reinserted later.
+        escaped_fmt = re.sub(self._ESCAPE_RE, "#" , fmt)
+        # Any number of S is the same as one.
+        escaped_fmt = re.sub('S+', 'S', escaped_fmt)
+        escaped_data = re.findall(self._ESCAPE_RE, fmt)
+
+        fmt_pattern = escaped_fmt
+
+        for m in self._FORMAT_RE.finditer(escaped_fmt):
             token = m.group(0)
             try:
                 input_re = self._input_re_map[token]
@@ -140,9 +143,20 @@ class DateTimeParser(object):
             # are returned in the order found by finditer.
             fmt_pattern = fmt_pattern[:m.start() + offset] + input_pattern + fmt_pattern[m.end() + offset:]
             offset += len(input_pattern) - (m.end() - m.start())
-        match = re.search(fmt_pattern, string, flags=re.IGNORECASE)
+
+        final_fmt_pattern = ""
+        a = fmt_pattern.split("#")
+        b = escaped_data
+
+        # Due to the way Python splits, 'a' will always be longer
+        for i in range(len(a)):
+            final_fmt_pattern += a[i]
+            if i < len(b):
+                final_fmt_pattern += b[i][1:-1]
+
+        match = re.search(final_fmt_pattern, string, flags=re.IGNORECASE)
         if match is None:
-            raise ParserError('Failed to match \'{0}\' when parsing \'{1}\''.format(fmt_pattern, string))
+            raise ParserError('Failed to match \'{0}\' when parsing \'{1}\''.format(final_fmt_pattern, string))
         parts = {}
         for token in tokens:
             if token == 'Do':
@@ -181,18 +195,22 @@ class DateTimeParser(object):
         elif token in ['ss', 's']:
             parts['second'] = int(value)
 
-        elif token == 'SSSSSS':
-            parts['microsecond'] = int(value)
-        elif token == 'SSSSS':
-            parts['microsecond'] = int(value) * 10
-        elif token == 'SSSS':
-            parts['microsecond'] = int(value) * 100
-        elif token == 'SSS':
-            parts['microsecond'] = int(value) * 1000
-        elif token == 'SS':
-            parts['microsecond'] = int(value) * 10000
         elif token == 'S':
-            parts['microsecond'] = int(value) * 100000
+            # We have the *most significant* digits of an arbitrary-precision integer.
+            # We want the six most significant digits as an integer, rounded.
+            # FIXME: add nanosecond support somehow?
+            value = value.ljust(7, str('0'))
+
+            # floating-point (IEEE-754) defaults to half-to-even rounding
+            seventh_digit = int(value[6])
+            if seventh_digit == 5:
+                rounding = int(value[5]) % 2
+            elif seventh_digit > 5:
+                rounding = 1
+            else:
+                rounding = 0
+
+            parts['microsecond'] = int(value[:6]) + rounding
 
         elif token == 'X':
             parts['timestamp'] = int(value)
@@ -242,7 +260,7 @@ class DateTimeParser(object):
             try:
                 _datetime = self.parse(string, fmt)
                 break
-            except:
+            except ParserError:
                 pass
 
         if _datetime is None:
@@ -273,7 +291,7 @@ class DateTimeParser(object):
 
 class TzinfoParser(object):
 
-    _TZINFO_RE = re.compile('([+\-])?(\d\d):?(\d\d)')
+    _TZINFO_RE = re.compile('([+\-])?(\d\d):?(\d\d)?')
 
     @classmethod
     def parse(cls, string):
@@ -292,6 +310,8 @@ class TzinfoParser(object):
 
             if iso_match:
                 sign, hours, minutes = iso_match.groups()
+                if minutes is None:
+                    minutes = 0
                 seconds = int(hours) * 3600 + int(minutes) * 60
 
                 if sign == '-':
@@ -303,6 +323,6 @@ class TzinfoParser(object):
                 tzinfo = tz.gettz(string)
 
         if tzinfo is None:
-            raise ParserError('Could not parse timezone expression "{0}"', string)
+            raise ParserError('Could not parse timezone expression "{0}"'.format(string))
 
         return tzinfo

lib/arrow/util.py

@@ -22,6 +22,8 @@ else: # pragma: no cover
     total_seconds = _total_seconds_27
 
 def is_timestamp(value):
+    if type(value) == bool:
+        return False
     try:
         float(value)
         return True

lib/funcsigs/__init__.py

@@ -0,0 +1,829 @@
+# Copyright 2001-2013 Python Software Foundation; All Rights Reserved
+"""Function signature objects for callables
+
+Back port of Python 3.3's function signature tools from the inspect module,
+modified to be compatible with Python 2.6, 2.7 and 3.3+.
+"""
+from __future__ import absolute_import, division, print_function
+import itertools
+import functools
+import re
+import types
+
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict
+
+from funcsigs.version import __version__
+
+__all__ = ['BoundArguments', 'Parameter', 'Signature', 'signature']
+
+
+_WrapperDescriptor = type(type.__call__)
+_MethodWrapper = type(all.__call__)
+
+_NonUserDefinedCallables = (_WrapperDescriptor,
+                            _MethodWrapper,
+                            types.BuiltinFunctionType)
+
+
+def formatannotation(annotation, base_module=None):
+    if isinstance(annotation, type):
+        if annotation.__module__ in ('builtins', '__builtin__', base_module):
+            return annotation.__name__
+        return annotation.__module__+'.'+annotation.__name__
+    return repr(annotation)
+
+
+def _get_user_defined_method(cls, method_name, *nested):
+    try:
+        if cls is type:
+            return
+        meth = getattr(cls, method_name)
+        for name in nested:
+            meth = getattr(meth, name, meth)
+    except AttributeError:
+        return
+    else:
+        if not isinstance(meth, _NonUserDefinedCallables):
+            # Once '__signature__' will be added to 'C'-level
+            # callables, this check won't be necessary
+            return meth
+
+
+def signature(obj):
+    '''Get a signature object for the passed callable.'''
+
+    if not callable(obj):
+        raise TypeError('{0!r} is not a callable object'.format(obj))
+
+    if isinstance(obj, types.MethodType):
+        sig = signature(obj.__func__)
+        if obj.__self__ is None:
+            # Unbound method - preserve as-is.
+            return sig
+        else:
+            # Bound method. Eat self - if we can.
+            params = tuple(sig.parameters.values())
+
+            if not params or params[0].kind in (_VAR_KEYWORD, _KEYWORD_ONLY):
+                raise ValueError('invalid method signature')
+
+            kind = params[0].kind
+            if kind in (_POSITIONAL_OR_KEYWORD, _POSITIONAL_ONLY):
+                # Drop first parameter:
+                # '(p1, p2[, ...])' -> '(p2[, ...])'
+                params = params[1:]
+            else:
+                if kind is not _VAR_POSITIONAL:
+                    # Unless we add a new parameter type we never
+                    # get here
+                    raise ValueError('invalid argument type')
+                # It's a var-positional parameter.
+                # Do nothing. '(*args[, ...])' -> '(*args[, ...])'
+
+            return sig.replace(parameters=params)
+
+    try:
+        sig = obj.__signature__
+    except AttributeError:
+        pass
+    else:
+        if sig is not None:
+            return sig
+
+    try:
+        # Was this function wrapped by a decorator?
+        wrapped = obj.__wrapped__
+    except AttributeError:
+        pass
+    else:
+        return signature(wrapped)
+
+    if isinstance(obj, types.FunctionType):
+        return Signature.from_function(obj)
+
+    if isinstance(obj, functools.partial):
+        sig = signature(obj.func)
+
+        new_params = OrderedDict(sig.parameters.items())
+
+        partial_args = obj.args or ()
+        partial_keywords = obj.keywords or {}
+        try:
+            ba = sig.bind_partial(*partial_args, **partial_keywords)
+        except TypeError as ex:
+            msg = 'partial object {0!r} has incorrect arguments'.format(obj)
+            raise ValueError(msg)
+
+        for arg_name, arg_value in ba.arguments.items():
+            param = new_params[arg_name]
+            if arg_name in partial_keywords:
+                # We set a new default value, because the following code
+                # is correct:
+                #
+                #   >>> def foo(a): print(a)
+                #   >>> print(partial(partial(foo, a=10), a=20)())
+                #   20
+                #   >>> print(partial(partial(foo, a=10), a=20)(a=30))
+                #   30
+                #
+                # So, with 'partial' objects, passing a keyword argument is
+                # like setting a new default value for the corresponding
+                # parameter
+                #
+                # We also mark this parameter with '_partial_kwarg'
+                # flag.  Later, in '_bind', the 'default' value of this
+                # parameter will be added to 'kwargs', to simulate
+                # the 'functools.partial' real call.
+                new_params[arg_name] = param.replace(default=arg_value,
+                                                     _partial_kwarg=True)
+
+            elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and
+                            not param._partial_kwarg):
+                new_params.pop(arg_name)
+
+        return sig.replace(parameters=new_params.values())
+
+    sig = None
+    if isinstance(obj, type):
+        # obj is a class or a metaclass
+
+        # First, let's see if it has an overloaded __call__ defined
+        # in its metaclass
+        call = _get_user_defined_method(type(obj), '__call__')
+        if call is not None:
+            sig = signature(call)
+        else:
+            # Now we check if the 'obj' class has a '__new__' method
+            new = _get_user_defined_method(obj, '__new__')
+            if new is not None:
+                sig = signature(new)
+            else:
+                # Finally, we should have at least __init__ implemented
+                init = _get_user_defined_method(obj, '__init__')
+                if init is not None:
+                    sig = signature(init)
+    elif not isinstance(obj, _NonUserDefinedCallables):
+        # An object with __call__
+        # We also check that the 'obj' is not an instance of
+        # _WrapperDescriptor or _MethodWrapper to avoid
+        # infinite recursion (and even potential segfault)
+        call = _get_user_defined_method(type(obj), '__call__', 'im_func')
+        if call is not None:
+            sig = signature(call)
+
+    if sig is not None:
+        # For classes and objects we skip the first parameter of their
+        # __call__, __new__, or __init__ methods
+        return sig.replace(parameters=tuple(sig.parameters.values())[1:])
+
+    if isinstance(obj, types.BuiltinFunctionType):
+        # Raise a nicer error message for builtins
+        msg = 'no signature found for builtin function {0!r}'.format(obj)
+        raise ValueError(msg)
+
+    raise ValueError('callable {0!r} is not supported by signature'.format(obj))
+
+
+class _void(object):
+    '''A private marker - used in Parameter & Signature'''
+
+
+class _empty(object):
+    pass
+
+
+class _ParameterKind(int):
+    def __new__(self, *args, **kwargs):
+        obj = int.__new__(self, *args)
+        obj._name = kwargs['name']
+        return obj
+
+    def __str__(self):
+        return self._name
+
+    def __repr__(self):
+        return '<_ParameterKind: {0!r}>'.format(self._name)
+
+
+_POSITIONAL_ONLY        = _ParameterKind(0, name='POSITIONAL_ONLY')
+_POSITIONAL_OR_KEYWORD  = _ParameterKind(1, name='POSITIONAL_OR_KEYWORD')
+_VAR_POSITIONAL         = _ParameterKind(2, name='VAR_POSITIONAL')
+_KEYWORD_ONLY           = _ParameterKind(3, name='KEYWORD_ONLY')
+_VAR_KEYWORD            = _ParameterKind(4, name='VAR_KEYWORD')
+
+
+class Parameter(object):
+    '''Represents a parameter in a function signature.
+
+    Has the following public attributes:
+
+    * name : str
+        The name of the parameter as a string.
+    * default : object
+        The default value for the parameter if specified.  If the
+        parameter has no default value, this attribute is not set.
+    * annotation
+        The annotation for the parameter if specified.  If the
+        parameter has no annotation, this attribute is not set.
+    * kind : str
+        Describes how argument values are bound to the parameter.
+        Possible values: `Parameter.POSITIONAL_ONLY`,
+        `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`,
+        `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`.
+    '''
+
+    __slots__ = ('_name', '_kind', '_default', '_annotation', '_partial_kwarg')
+
+    POSITIONAL_ONLY         = _POSITIONAL_ONLY
+    POSITIONAL_OR_KEYWORD   = _POSITIONAL_OR_KEYWORD
+    VAR_POSITIONAL          = _VAR_POSITIONAL
+    KEYWORD_ONLY            = _KEYWORD_ONLY
+    VAR_KEYWORD             = _VAR_KEYWORD
+
+    empty = _empty
+
+    def __init__(self, name, kind, default=_empty, annotation=_empty,
+                 _partial_kwarg=False):
+
+        if kind not in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD,
+                        _VAR_POSITIONAL, _KEYWORD_ONLY, _VAR_KEYWORD):
+            raise ValueError("invalid value for 'Parameter.kind' attribute")
+        self._kind = kind
+
+        if default is not _empty:
+            if kind in (_VAR_POSITIONAL, _VAR_KEYWORD):
+                msg = '{0} parameters cannot have default values'.format(kind)
+                raise ValueError(msg)
+        self._default = default
+        self._annotation = annotation
+
+        if name is None:
+            if kind != _POSITIONAL_ONLY:
+                raise ValueError("None is not a valid name for a "
+                                 "non-positional-only parameter")
+            self._name = name
+        else:
+            name = str(name)
+            if kind != _POSITIONAL_ONLY and not re.match(r'[a-z_]\w*$', name, re.I):
+                msg = '{0!r} is not a valid parameter name'.format(name)
+                raise ValueError(msg)
+            self._name = name
+
+        self._partial_kwarg = _partial_kwarg
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def default(self):
+        return self._default
+
+    @property
+    def annotation(self):
+        return self._annotation
+
+    @property
+    def kind(self):
+        return self._kind
+
+    def replace(self, name=_void, kind=_void, annotation=_void,
+                default=_void, _partial_kwarg=_void):
+        '''Creates a customized copy of the Parameter.'''
+
+        if name is _void:
+            name = self._name
+
+        if kind is _void:
+            kind = self._kind
+
+        if annotation is _void:
+            annotation = self._annotation
+
+        if default is _void:
+            default = self._default
+
+        if _partial_kwarg is _void:
+            _partial_kwarg = self._partial_kwarg
+
+        return type(self)(name, kind, default=default, annotation=annotation,
+                          _partial_kwarg=_partial_kwarg)
+
+    def __str__(self):
+        kind = self.kind
+
+        formatted = self._name
+        if kind == _POSITIONAL_ONLY:
+            if formatted is None:
+                formatted = ''
+            formatted = '<{0}>'.format(formatted)
+
+        # Add annotation and default value
+        if self._annotation is not _empty:
+            formatted = '{0}:{1}'.format(formatted,
+                                       formatannotation(self._annotation))
+
+        if self._default is not _empty:
+            formatted = '{0}={1}'.format(formatted, repr(self._default))
+
+        if kind == _VAR_POSITIONAL:
+            formatted = '*' + formatted
+        elif kind == _VAR_KEYWORD:
+            formatted = '**' + formatted
+
+        return formatted
+
+    def __repr__(self):
+        return '<{0} at {1:#x} {2!r}>'.format(self.__class__.__name__,
+                                           id(self), self.name)
+
+    def __hash__(self):
+        msg = "unhashable type: '{0}'".format(self.__class__.__name__)
+        raise TypeError(msg)
+
+    def __eq__(self, other):
+        return (issubclass(other.__class__, Parameter) and
+                self._name == other._name and
+                self._kind == other._kind and
+                self._default == other._default and
+                self._annotation == other._annotation)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class BoundArguments(object):
+    '''Result of `Signature.bind` call.  Holds the mapping of arguments
+    to the function's parameters.
+
+    Has the following public attributes:
+
+    * arguments : OrderedDict
+        An ordered mutable mapping of parameters' names to arguments' values.
+        Does not contain arguments' default values.
+    * signature : Signature
+        The Signature object that created this instance.
+    * args : tuple
+        Tuple of positional arguments values.
+    * kwargs : dict
+        Dict of keyword arguments values.
+    '''
+
+    def __init__(self, signature, arguments):
+        self.arguments = arguments
+        self._signature = signature
+
+    @property
+    def signature(self):
+        return self._signature
+
+    @property
+    def args(self):
+        args = []
+        for param_name, param in self._signature.parameters.items():
+            if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
+                                                    param._partial_kwarg):
+                # Keyword arguments mapped by 'functools.partial'
+                # (Parameter._partial_kwarg is True) are mapped
+                # in 'BoundArguments.kwargs', along with VAR_KEYWORD &
+                # KEYWORD_ONLY
+                break
+
+            try:
+                arg = self.arguments[param_name]
+            except KeyError:
+                # We're done here. Other arguments
+                # will be mapped in 'BoundArguments.kwargs'
+                break
+            else:
+                if param.kind == _VAR_POSITIONAL:
+                    # *args
+                    args.extend(arg)
+                else:
+                    # plain argument
+                    args.append(arg)
+
+        return tuple(args)
+
+    @property
+    def kwargs(self):
+        kwargs = {}
+        kwargs_started = False
+        for param_name, param in self._signature.parameters.items():
+            if not kwargs_started:
+                if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or
+                                                param._partial_kwarg):
+                    kwargs_started = True
+                else:
+                    if param_name not in self.arguments:
+                        kwargs_started = True
+                        continue
+
+            if not kwargs_started:
+                continue
+
+            try:
+                arg = self.arguments[param_name]
+            except KeyError:
+                pass
+            else:
+                if param.kind == _VAR_KEYWORD:
+                    # **kwargs
+                    kwargs.update(arg)
+                else:
+                    # plain keyword argument
+                    kwargs[param_name] = arg
+
+        return kwargs
+
+    def __hash__(self):
+        msg = "unhashable type: '{0}'".format(self.__class__.__name__)
+        raise TypeError(msg)
+
+    def __eq__(self, other):
+        return (issubclass(other.__class__, BoundArguments) and
+                self.signature == other.signature and
+                self.arguments == other.arguments)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class Signature(object):
+    '''A Signature object represents the overall signature of a function.
+    It stores a Parameter object for each parameter accepted by the
+    function, as well as information specific to the function itself.
+
+    A Signature object has the following public attributes and methods:
+
+    * parameters : OrderedDict
+        An ordered mapping of parameters' names to the corresponding
+        Parameter objects (keyword-only arguments are in the same order
+        as listed in `code.co_varnames`).
+    * return_annotation : object
+        The annotation for the return type of the function if specified.
+        If the function has no annotation for its return type, this
+        attribute is not set.
+    * bind(*args, **kwargs) -> BoundArguments
+        Creates a mapping from positional and keyword arguments to
+        parameters.
+    * bind_partial(*args, **kwargs) -> BoundArguments
+        Creates a partial mapping from positional and keyword arguments
+        to parameters (simulating 'functools.partial' behavior.)
+    '''
+
+    __slots__ = ('_return_annotation', '_parameters')
+
+    _parameter_cls = Parameter
+    _bound_arguments_cls = BoundArguments
+
+    empty = _empty
+
+    def __init__(self, parameters=None, return_annotation=_empty,
+                 __validate_parameters__=True):
+        '''Constructs Signature from the given list of Parameter
+        objects and 'return_annotation'.  All arguments are optional.
+        '''
+
+        if parameters is None:
+            params = OrderedDict()
+        else:
+            if __validate_parameters__:
+                params = OrderedDict()
+                top_kind = _POSITIONAL_ONLY
+
+                for idx, param in enumerate(parameters):
+                    kind = param.kind
+                    if kind < top_kind:
+                        msg = 'wrong parameter order: {0} before {1}'
+                        msg = msg.format(top_kind, param.kind)
+                        raise ValueError(msg)
+                    else:
+                        top_kind = kind
+
+                    name = param.name
+                    if name is None:
+                        name = str(idx)
+                        param = param.replace(name=name)
+
+                    if name in params:
+                        msg = 'duplicate parameter name: {0!r}'.format(name)
+                        raise ValueError(msg)
+                    params[name] = param
+            else:
+                params = OrderedDict(((param.name, param)
+                                                for param in parameters))
+
+        self._parameters = params
+        self._return_annotation = return_annotation
+
+    @classmethod
+    def from_function(cls, func):
+        '''Constructs Signature for the given python function'''
+
+        if not isinstance(func, types.FunctionType):
+            raise TypeError('{0!r} is not a Python function'.format(func))
+
+        Parameter = cls._parameter_cls
+
+        # Parameter information.
+        func_code = func.__code__
+        pos_count = func_code.co_argcount
+        arg_names = func_code.co_varnames
+        positional = tuple(arg_names[:pos_count])
+        keyword_only_count = getattr(func_code, 'co_kwonlyargcount', 0)
+        keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)]
+        annotations = getattr(func, '__annotations__', {})
+        defaults = func.__defaults__
+        kwdefaults = getattr(func, '__kwdefaults__', None)
+
+        if defaults:
+            pos_default_count = len(defaults)
+        else:
+            pos_default_count = 0
+
+        parameters = []
+
+        # Non-keyword-only parameters w/o defaults.
+        non_default_count = pos_count - pos_default_count
+        for name in positional[:non_default_count]:
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_POSITIONAL_OR_KEYWORD))
+
+        # ... w/ defaults.
+        for offset, name in enumerate(positional[non_default_count:]):
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_POSITIONAL_OR_KEYWORD,
+                                        default=defaults[offset]))
+
+        # *args
+        if func_code.co_flags & 0x04:
+            name = arg_names[pos_count + keyword_only_count]
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_VAR_POSITIONAL))
+
+        # Keyword-only parameters.
+        for name in keyword_only:
+            default = _empty
+            if kwdefaults is not None:
+                default = kwdefaults.get(name, _empty)
+
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_KEYWORD_ONLY,
+                                        default=default))
+        # **kwargs
+        if func_code.co_flags & 0x08:
+            index = pos_count + keyword_only_count
+            if func_code.co_flags & 0x04:
+                index += 1
+
+            name = arg_names[index]
+            annotation = annotations.get(name, _empty)
+            parameters.append(Parameter(name, annotation=annotation,
+                                        kind=_VAR_KEYWORD))
+
+        return cls(parameters,
+                   return_annotation=annotations.get('return', _empty),
+                   __validate_parameters__=False)
+
+    @property
+    def parameters(self):
+        try:
+            return types.MappingProxyType(self._parameters)
+        except AttributeError:
+            return OrderedDict(self._parameters.items())
+
+    @property
+    def return_annotation(self):
+        return self._return_annotation
+
+    def replace(self, parameters=_void, return_annotation=_void):
+        '''Creates a customized copy of the Signature.
+        Pass 'parameters' and/or 'return_annotation' arguments
+        to override them in the new copy.
+        '''
+
+        if parameters is _void:
+            parameters = self.parameters.values()
+
+        if return_annotation is _void:
+            return_annotation = self._return_annotation
+
+        return type(self)(parameters,
+                          return_annotation=return_annotation)
+
+    def __hash__(self):
+        msg = "unhashable type: '{0}'".format(self.__class__.__name__)
+        raise TypeError(msg)
+
+    def __eq__(self, other):
+        if (not issubclass(type(other), Signature) or
+                    self.return_annotation != other.return_annotation or
+                    len(self.parameters) != len(other.parameters)):
+            return False
+
+        other_positions = dict((param, idx)
+                           for idx, param in enumerate(other.parameters.keys()))
+
+        for idx, (param_name, param) in enumerate(self.parameters.items()):
+            if param.kind == _KEYWORD_ONLY:
+                try:
+                    other_param = other.parameters[param_name]
+                except KeyError:
+                    return False
+                else:
+                    if param != other_param:
+                        return False
+            else:
+                try:
+                    other_idx = other_positions[param_name]
+                except KeyError:
+                    return False
+                else:
+                    if (idx != other_idx or
+                                    param != other.parameters[param_name]):
+                        return False
+
+        return True
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def _bind(self, args, kwargs, partial=False):
+        '''Private method.  Don't use directly.'''
+
+        arguments = OrderedDict()
+
+        parameters = iter(self.parameters.values())
+        parameters_ex = ()
+        arg_vals = iter(args)
+
+        if partial:
+            # Support for binding arguments to 'functools.partial' objects.
+            # See 'functools.partial' case in 'signature()' implementation
+            # for details.
+            for param_name, param in self.parameters.items():
+                if (param._partial_kwarg and param_name not in kwargs):
+                    # Simulating 'functools.partial' behavior
+                    kwargs[param_name] = param.default
+
+        while True:
+            # Let's iterate through the positional arguments and corresponding
+            # parameters
+            try:
+                arg_val = next(arg_vals)
+            except StopIteration:
+                # No more positional arguments
+                try:
+                    param = next(parameters)
+                except StopIteration:
+                    # No more parameters. That's it. Just need to check that
+                    # we have no `kwargs` after this while loop
+                    break
+                else:
+                    if param.kind == _VAR_POSITIONAL:
+                        # That's OK, just empty *args.  Let's start parsing
+                        # kwargs
+                        break
+                    elif param.name in kwargs:
+                        if param.kind == _POSITIONAL_ONLY:
+                            msg = '{arg!r} parameter is positional only, ' \
+                                  'but was passed as a keyword'
+                            msg = msg.format(arg=param.name)
+                            raise TypeError(msg)
+                        parameters_ex = (param,)
+                        break
+                    elif (param.kind == _VAR_KEYWORD or
+                                                param.default is not _empty):
+                        # That's fine too - we have a default value for this
+                        # parameter.  So, lets start parsing `kwargs`, starting
+                        # with the current parameter
+                        parameters_ex = (param,)
+                        break
+                    else:
+                        if partial:
+                            parameters_ex = (param,)
+                            break
+                        else:
+                            msg = '{arg!r} parameter lacking default value'
+                            msg = msg.format(arg=param.name)
+                            raise TypeError(msg)
+            else:
+                # We have a positional argument to process
+                try:
+                    param = next(parameters)
+                except StopIteration:
+                    raise TypeError('too many positional arguments')
+                else:
+                    if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY):
+                        # Looks like we have no parameter for this positional
+                        # argument
+                        raise TypeError('too many positional arguments')
+
+                    if param.kind == _VAR_POSITIONAL:
+                        # We have an '*args'-like argument, let's fill it with
+                        # all positional arguments we have left and move on to
+                        # the next phase
+                        values = [arg_val]
+                        values.extend(arg_vals)
+                        arguments[param.name] = tuple(values)
+                        break
+
+                    if param.name in kwargs:
+                        raise TypeError('multiple values for argument '
+                                        '{arg!r}'.format(arg=param.name))
+
+                    arguments[param.name] = arg_val
+
+        # Now, we iterate through the remaining parameters to process
+        # keyword arguments
+        kwargs_param = None
+        for param in itertools.chain(parameters_ex, parameters):
+            if param.kind == _POSITIONAL_ONLY:
+                # This should never happen in case of a properly built
+                # Signature object (but let's have this check here
+                # to ensure correct behaviour just in case)
+                raise TypeError('{arg!r} parameter is positional only, '
+                                'but was passed as a keyword'. \
+                                format(arg=param.name))
+
+            if param.kind == _VAR_KEYWORD:
+                # Memorize that we have a '**kwargs'-like parameter
+                kwargs_param = param
+                continue
+
+            param_name = param.name
+            try:
+                arg_val = kwargs.pop(param_name)
+            except KeyError:
+                # We have no value for this parameter.  It's fine though,
+                # if it has a default value, or it is an '*args'-like
+                # parameter, left alone by the processing of positional
+                # arguments.
+                if (not partial and param.kind != _VAR_POSITIONAL and
+                                                    param.default is _empty):
+                    raise TypeError('{arg!r} parameter lacking default value'. \
+                                    format(arg=param_name))
+
+            else:
+                arguments[param_name] = arg_val
+
+        if kwargs:
+            if kwargs_param is not None:
+                # Process our '**kwargs'-like parameter
+                arguments[kwargs_param.name] = kwargs
+            else:
+                raise TypeError('too many keyword arguments %r' % kwargs)
+
+        return self._bound_arguments_cls(self, arguments)
+
+    def bind(*args, **kwargs):
+        '''Get a BoundArguments object, that maps the passed `args`
+        and `kwargs` to the function's signature.  Raises `TypeError`
+        if the passed arguments can not be bound.
+        '''
+        return args[0]._bind(args[1:], kwargs)
+
+    def bind_partial(self, *args, **kwargs):
+        '''Get a BoundArguments object, that partially maps the
+        passed `args` and `kwargs` to the function's signature.
+        Raises `TypeError` if the passed arguments can not be bound.
+        '''
+        return self._bind(args, kwargs, partial=True)
+
+    def __str__(self):
+        result = []
+        render_kw_only_separator = True
+        for idx, param in enumerate(self.parameters.values()):
+            formatted = str(param)
+
+            kind = param.kind
+            if kind == _VAR_POSITIONAL:
+                # OK, we have an '*args'-like parameter, so we won't need
+                # a '*' to separate keyword-only arguments
+                render_kw_only_separator = False
+            elif kind == _KEYWORD_ONLY and render_kw_only_separator:
+                # We have a keyword-only parameter to render and we haven't
+                # rendered an '*args'-like parameter before, so add a '*'
+                # separator to the parameters list ("foo(arg1, *, arg2)" case)
+                result.append('*')
+                # This condition should be only triggered once, so
+                # reset the flag
+                render_kw_only_separator = False
+
+            result.append(formatted)
+
+        rendered = '({0})'.format(', '.join(result))
+
+        if self.return_annotation is not _empty:
+            anno = formatannotation(self.return_annotation)
+            rendered += ' -> {0}'.format(anno)
+
+        return rendered

lib/funcsigs/version.py

@@ -0,0 +1 @@
+__version__ = "1.0.2"

lib/ratelimit/__init__.py

@@ -0,0 +1,59 @@
+from math import floor
+
+import time
+import sys
+import threading
+import functools
+
+
+def clamp(value):
+    '''
+    Clamp integer between 1 and max
+
+    There must be at least 1 method invocation
+    made over the time period. Make sure the
+    value passed is at least 1 and is not a
+    fraction of an invocation.
+
+    :param float value: The number of method invocations.
+    :return: Clamped number of invocations.
+    :rtype: int
+    '''
+    return max(1, min(sys.maxsize, floor(value)))
+
+
+class RateLimitDecorator:
+    def __init__(self, period=1, every=1.0):
+        self.frequency = abs(every) / float(clamp(period))
+        self.last_called = 0.0
+        self.lock = threading.RLock()
+
+    def __call__(self, func):
+        '''
+        Extend the behaviour of the following
+        function, forwarding method invocations
+        if the time window hes elapsed.
+
+        :param function func: The function to decorate.
+        :return: Decorated function.
+        :rtype: function
+        '''
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            '''Decorator wrapper function'''
+            with self.lock:
+                elapsed = time.time() - self.last_called
+                left_to_wait = self.frequency - elapsed
+                if left_to_wait > 0:
+                    time.sleep(left_to_wait)
+                self.last_called = time.time()
+            return func(*args, **kwargs)
+        return wrapper
+
+
+rate_limited = RateLimitDecorator
+
+
+__all__ = [
+    'rate_limited'
+]

lib/ratelimit/version.py

@@ -0,0 +1,9 @@
+class Version(object):
+    '''Version of the package'''
+
+    def __setattr__(self, *args):
+        raise TypeError('cannot modify immutable instance')
+    __delattr__ = __setattr__
+
+    def __init__(self, num):
+        super(Version, self).__setattr__('number', num)

plexpy/__init__.py

@@ -38,9 +38,11 @@ import activity_handler
 import activity_pinger
 import common
 import database
+import datafactory
 import libraries
 import logger
 import mobile_app
+import newsletter_handler
 import notification_handler
 import notifiers
 import plextv
@@ -166,6 +168,14 @@ def initialize(config_file):
             except OSError as e:
                 logger.error(u"Could not create cache dir '%s': %s" % (CONFIG.CACHE_DIR, e))
 
+        if not CONFIG.NEWSLETTER_DIR:
+            CONFIG.NEWSLETTER_DIR = os.path.join(DATA_DIR, 'newsletters')
+        if not os.path.exists(CONFIG.NEWSLETTER_DIR):
+            try:
+                os.makedirs(CONFIG.NEWSLETTER_DIR)
+            except OSError as e:
+                logger.error(u"Could not create newsletter dir '%s': %s" % (CONFIG.NEWSLETTER_DIR, e))
+
         # Initialize the database
         logger.info(u"Checking if the database upgrades are required...")
         try:
@@ -426,7 +436,7 @@ def initialize_scheduler():
                 logger.error(e)
 
 
-def schedule_job(function, name, hours=0, minutes=0, seconds=0, args=None):
+def schedule_job(func, name, hours=0, minutes=0, seconds=0, args=None):
     """
     Start scheduled job if starting or restarting plexpy.
     Reschedule job if Interval Settings have changed.
@@ -444,7 +454,7 @@ def schedule_job(function, name, hours=0, minutes=0, seconds=0, args=None):
                 hours=hours, minutes=minutes, seconds=seconds), args=args)
             logger.info(u"Re-scheduled background task: %s", name)
     elif hours > 0 or minutes > 0 or seconds > 0:
-        SCHED.add_job(function, id=name, trigger=IntervalTrigger(
+        SCHED.add_job(func, id=name, trigger=IntervalTrigger(
             hours=hours, minutes=minutes, seconds=seconds), args=args)
         logger.info(u"Scheduled background task: %s", name)
 
@@ -476,6 +486,10 @@ def start():
 
             analytics_event(category='system', action='start')
 
+        # Schedule newsletters
+        newsletter_handler.NEWSLETTER_SCHED.start()
+        newsletter_handler.schedule_newsletters()
+
         _STARTED = True
 
 
@@ -515,8 +529,8 @@ def dbcheck():
         'transcode_hw_decoding INTEGER, transcode_hw_encoding INTEGER, '
         'optimized_version INTEGER, optimized_version_profile TEXT, optimized_version_title TEXT, '
         'synced_version INTEGER, synced_version_profile TEXT, '
-        'buffer_count INTEGER DEFAULT 0, buffer_last_triggered INTEGER, last_paused INTEGER, write_attempts INTEGER DEFAULT 0, '
-        'raw_stream_info TEXT)'
+        'buffer_count INTEGER DEFAULT 0, buffer_last_triggered INTEGER, last_paused INTEGER, watched INTEGER DEFAULT 0, '
+        'write_attempts INTEGER DEFAULT 0, raw_stream_info TEXT)'
     )
 
     # session_history table :: This is a history table which logs essential stream details
@@ -574,14 +588,6 @@ def dbcheck():
         'filter_music TEXT, filter_photos TEXT)'
     )
 
-    # notify_log table :: This is a table which logs notifications sent
-    c_db.execute(
-        'CREATE TABLE IF NOT EXISTS notify_log (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER, '
-        'session_key INTEGER, rating_key INTEGER, parent_rating_key INTEGER, grandparent_rating_key INTEGER, '
-        'user_id INTEGER, user TEXT, notifier_id INTEGER, agent_id INTEGER, agent_name TEXT, notify_action TEXT, '
-        'subject_text TEXT, body_text TEXT, script_args TEXT, success INTEGER DEFAULT 0)'
-    )
-
     # library_sections table :: This table keeps record of the servers library sections
     c_db.execute(
         'CREATE TABLE IF NOT EXISTS library_sections (id INTEGER PRIMARY KEY AUTOINCREMENT, '
@@ -620,10 +626,29 @@ def dbcheck():
         'custom_conditions TEXT, custom_conditions_logic TEXT)'
     )
 
-    # poster_urls table :: This table keeps record of the notification poster urls
+    # notify_log table :: This is a table which logs notifications sent
     c_db.execute(
-        'CREATE TABLE IF NOT EXISTS poster_urls (id INTEGER PRIMARY KEY AUTOINCREMENT, '
-        'rating_key INTEGER, poster_title TEXT, poster_url TEXT, delete_hash TEXT)'
+        'CREATE TABLE IF NOT EXISTS notify_log (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER, '
+        'session_key INTEGER, rating_key INTEGER, parent_rating_key INTEGER, grandparent_rating_key INTEGER, '
+        'user_id INTEGER, user TEXT, notifier_id INTEGER, agent_id INTEGER, agent_name TEXT, notify_action TEXT, '
+        'subject_text TEXT, body_text TEXT, script_args TEXT, success INTEGER DEFAULT 0)'
+    )
+
+    # newsletters table :: This table keeps record of the newsletter settings
+    c_db.execute(
+        'CREATE TABLE IF NOT EXISTS newsletters (id INTEGER PRIMARY KEY AUTOINCREMENT, '
+        'agent_id INTEGER, agent_name TEXT, agent_label TEXT, '
+        'friendly_name TEXT, newsletter_config TEXT, email_config TEXT, '
+        'subject TEXT, body TEXT, message TEXT, '
+        'cron TEXT NOT NULL DEFAULT "0 0 * * 0", active INTEGER DEFAULT 0)'
+    )
+
+    # newsletter_log table :: This is a table which logs newsletters sent
+    c_db.execute(
+        'CREATE TABLE IF NOT EXISTS newsletter_log (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp INTEGER, '
+        'newsletter_id INTEGER, agent_id INTEGER, agent_name TEXT, notify_action TEXT, '
+        'subject_text TEXT, body_text TEXT, message_text TEXT, start_date TEXT, end_date TEXT, '
+        'uuid TEXT UNIQUE, success INTEGER DEFAULT 0)'
     )
 
     # recently_added table :: This table keeps record of recently added items
@@ -655,6 +680,19 @@ def dbcheck():
         'themoviedb_id INTEGER, themoviedb_url TEXT, themoviedb_json TEXT)'
     )
 
+    # image_hash_lookup table :: This table keeps record of the image hash lookups
+    c_db.execute(
+        'CREATE TABLE IF NOT EXISTS image_hash_lookup (id INTEGER PRIMARY KEY AUTOINCREMENT, '
+        'img_hash TEXT, img TEXT, rating_key INTEGER, width INTEGER, height INTEGER, '
+        'opacity INTEGER, background TEXT, blur INTEGER, fallback TEXT)'
+    )
+
+    # imgur_lookup table :: This table keeps record of the Imgur uploads
+    c_db.execute(
+        'CREATE TABLE IF NOT EXISTS imgur_lookup (id INTEGER PRIMARY KEY AUTOINCREMENT, '
+        'img_hash TEXT, imgur_title TEXT, imgur_url TEXT, delete_hash TEXT)'
+    )
+
     # Upgrade sessions table from earlier versions
     try:
         c_db.execute('SELECT started FROM sessions')
@@ -1000,6 +1038,15 @@ def dbcheck():
             'ALTER TABLE sessions ADD COLUMN transcode_hw_encoding INTEGER'
         )
 
+    # Upgrade sessions table from earlier versions
+    try:
+        c_db.execute('SELECT watched FROM sessions')
+    except sqlite3.OperationalError:
+        logger.debug(u"Altering database. Updating database table sessions.")
+        c_db.execute(
+            'ALTER TABLE sessions ADD COLUMN watched INTEGER DEFAULT 0'
+        )
+
     # Upgrade session_history table from earlier versions
     try:
         c_db.execute('SELECT reference_id FROM session_history')
@@ -1587,15 +1634,6 @@ def dbcheck():
             'ALTER TABLE user_login ADD COLUMN success INTEGER DEFAULT 1'
         )
 
-    # Upgrade poster_urls table from earlier versions
-    try:
-        c_db.execute('SELECT delete_hash FROM poster_urls')
-    except sqlite3.OperationalError:
-        logger.debug(u"Altering database. Updating database table poster_urls.")
-        c_db.execute(
-            'ALTER TABLE poster_urls ADD COLUMN delete_hash TEXT'
-        )
-
     # Rename notifiers in the database
     result = c_db.execute('SELECT agent_label FROM notifiers '
                           'WHERE agent_label = "XBMC" OR agent_label = "OSX Notify"').fetchone()
@@ -1625,6 +1663,26 @@ def dbcheck():
     conn_db.commit()
     c_db.close()
 
+    # Migrate poster_urls to imgur_lookup table
+    try:
+        db = database.MonitorDatabase()
+        result = db.select('SELECT SQL FROM sqlite_master WHERE type="table" AND name="poster_urls"')
+        if result:
+            result = db.select('SELECT * FROM poster_urls')
+            logger.debug(u"Altering database. Updating database table imgur_lookup.")
+
+            data_factory = datafactory.DataFactory()
+
+            for row in result:
+                img_hash = notification_handler.set_hash_image_info(
+                    rating_key=row['rating_key'], width=600, height=1000, fallback='poster')
+                data_factory.set_imgur_info(img_hash=img_hash, imgur_title=row['poster_title'],
+                                            imgur_url=row['poster_url'], delete_hash=row['delete_hash'])
+
+            db.action('DROP TABLE poster_urls')
+    except sqlite3.OperationalError:
+        pass
+
 
 def upgrade():
     if CONFIG.UPDATE_NOTIFIERS_DB:

plexpy/activity_handler.py

@@ -271,7 +271,7 @@ class ActivityHandler(object):
 
                 # Monitor if the stream has reached the watch percentage for notifications
                 # The only purpose of this is for notifications
-                if this_state != 'buffering':
+                if not db_session['watched'] and this_state != 'buffering':
                     progress_percent = helpers.get_percent(self.timeline['viewOffset'], db_session['duration'])
                     watched_percent = {'movie': plexpy.CONFIG.MOVIE_WATCHED_PERCENT,
                                        'episode': plexpy.CONFIG.TV_WATCHED_PERCENT,
@@ -280,12 +280,12 @@ class ActivityHandler(object):
                                        }
 
                     if progress_percent >= watched_percent.get(db_session['media_type'], 101):
-                        watched_notifiers = notification_handler.get_notify_state_enabled(
-                            session=db_session, notify_action='on_watched', notified=False)
-
-                        if watched_notifiers:
                         logger.debug(u"Tautulli ActivityHandler :: Session %s watched."
                                      % str(self.get_session_key()))
+                        ap.set_watched(session_key=self.get_session_key())
+
+                        watched_notifiers = notification_handler.get_notify_state_enabled(
+                            session=db_session, notify_action='on_watched', notified=False)
 
                         for d in watched_notifiers:
                             plexpy.NOTIFY_QUEUE.put({'stream_data': db_session.copy(),

plexpy/activity_processor.py

@@ -540,3 +540,8 @@ class ActivityProcessor(object):
             session = self.get_session_by_key(session_key=session_key)
             self.db.action('UPDATE sessions SET write_attempts = ? WHERE session_key = ?',
                            [session['write_attempts'] + 1, session_key])
+
+    def set_watched(self, session_key=None):
+        self.db.action('UPDATE sessions SET watched = ?'
+                       'WHERE session_key = ?',
+                       [1, session_key])

plexpy/common.py

@@ -31,6 +31,10 @@ DEFAULT_POSTER_THUMB = "interfaces/default/images/poster.png"
 DEFAULT_COVER_THUMB = "interfaces/default/images/cover.png"
 DEFAULT_ART = "interfaces/default/images/art.png"
 
+ONLINE_POSTER_THUMB = "http://tautulli.com/images/poster.png"
+ONLINE_COVER_THUMB = "http://tautulli.com/images/cover.png"
+ONLINE_ART = "http://tautulli.com/images/art.png"
+
 MEDIA_TYPE_HEADERS = {
     'movie': 'Movies',
     'show': 'TV Shows',
@@ -499,7 +503,7 @@ NOTIFICATION_PARAMETERS = [
         'category': 'Tautulli Update Available',
         'parameters': [
              {'name': 'Tautulli Update Version', 'type': 'str', 'value': 'tautulli_update_version', 'description': 'The available update version for Tautulli.'},
-             {'name': 'Tautulli Update Release URL', 'type': 'str', 'value': 'tautulli_update_release_url', 'description': 'The release page URL on GitHub'},
+             {'name': 'Tautulli Update Release URL', 'type': 'str', 'value': 'tautulli_update_release_url', 'description': 'The release page URL on GitHub.'},
              {'name': 'Tautulli Update Tar', 'type': 'str', 'value': 'tautulli_update_tar', 'description': 'The tar download URL for the available update.'},
              {'name': 'Tautulli Update Zip', 'type': 'str', 'value': 'tautulli_update_zip', 'description': 'The zip download URL for the available update.'},
              {'name': 'Tautulli Update Commit', 'type': 'str', 'value': 'tautulli_update_commit', 'description': 'The commit hash for the available update.'},
@@ -508,3 +512,23 @@ NOTIFICATION_PARAMETERS = [
         ]
      },
 ]
+
+NEWSLETTER_PARAMETERS = [
+    {
+        'category': 'Global',
+        'parameters': [
+            {'name': 'Server Name', 'type': 'str', 'value': 'server_name', 'description': 'The name of your Plex Server.'},
+            {'name': 'Start Date', 'type': 'str', 'value': 'start_date', 'description': 'The start date of the newesletter.'},
+            {'name': 'End Date', 'type': 'str', 'value': 'end_date', 'description': 'The end date of the newesletter.'},
+            {'name': 'Newsletter Days', 'type': 'int', 'value': 'newsletter_days', 'description': 'The past number of days included in the newsletter.'},
+            {'name': 'Newsletter URL', 'type': 'str', 'value': 'newsletter_url', 'description': 'The self-hosted URL to the newsletter.'},
+            {'name': 'Newsletter UUID', 'type': 'str', 'value': 'newsletter_uuid', 'description': 'The unique identifier for the newsletter.'},
+        ]
+     },
+    {
+        'category': 'Recently Added',
+        'parameters': [
+            {'name': 'Included Libraries', 'type': 'str', 'value': 'newsletter_libraries', 'description': 'The list of libraries included in the newsletter.'},
+        ]
+    }
+]

plexpy/config.py

@@ -227,6 +227,7 @@ _CONFIG_DEFINITIONS = {
     'HTTP_ROOT': (str, 'General', ''),
     'HTTP_USERNAME': (str, 'General', ''),
     'HTTP_PLEX_ADMIN': (int, 'General', 0),
+    'HTTP_BASE_URL': (str, 'General', ''),
     'HIPCHAT_URL': (str, 'Hipchat', ''),
     'HIPCHAT_COLOR': (str, 'Hipchat', ''),
     'HIPCHAT_INCL_SUBJECT': (int, 'Hipchat', 1),
@@ -308,6 +309,9 @@ _CONFIG_DEFINITIONS = {
     'MONITOR_REMOTE_ACCESS': (int, 'Monitoring', 0),
     'MONITORING_INTERVAL': (int, 'Monitoring', 60),
     'MONITORING_USE_WEBSOCKET': (int, 'Monitoring', 0),
+    'NEWSLETTER_TEMPLATES': (str, 'Newsletter', 'newsletters'),
+    'NEWSLETTER_DIR': (str, 'Newsletter', ''),
+    'NEWSLETTER_SELF_HOSTED': (int, 'Newsletter', 0),
     'NMA_APIKEY': (str, 'NMA', ''),
     'NMA_ENABLED': (int, 'NMA', 0),
     'NMA_PRIORITY': (int, 'NMA', 0),
@@ -656,7 +660,7 @@ def make_backup(cleanup=False, scheduler=False):
         logger.debug(u"Tautulli Config :: Successfully backed up %s to %s" % (plexpy.CONFIG_FILE, backup_file))
         return True
     else:
-        logger.warn(u"Tautulli Config :: Failed to backup %s to %s" % (plexpy.CONFIG_FILE, backup_file))
+        logger.error(u"Tautulli Config :: Failed to backup %s to %s" % (plexpy.CONFIG_FILE, backup_file))
         return False
 
 

plexpy/database.py

@@ -94,7 +94,7 @@ def make_backup(cleanup=False, scheduler=False):
         logger.debug(u"Tautulli Database :: Successfully backed up %s to %s" % (db_filename(), backup_file))
         return True
     else:
-        logger.warn(u"Tautulli Database :: Failed to backup %s to %s" % (db_filename(), backup_file))
+        logger.error(u"Tautulli Database :: Failed to backup %s to %s" % (db_filename(), backup_file))
         return False
 
 

plexpy/datafactory.py

@@ -1132,9 +1132,94 @@ class DataFactory(object):
 
         return ip_address
 
-    def get_poster_info(self, rating_key='', metadata=None):
+    def get_imgur_info(self, img=None, rating_key=None, width=None, height=None,
+                       opacity=None, background=None, blur=None, fallback=None,
+                       order_by=''):
         monitor_db = database.MonitorDatabase()
 
+        imgur_info = []
+
+        where_params = []
+        args = []
+
+        if img is not None:
+            where_params.append('img')
+            args.append(img)
+        if rating_key is not None:
+            where_params.append('rating_key')
+            args.append(rating_key)
+        if width is not None:
+            where_params.append('width')
+            args.append(width)
+        if height is not None:
+            where_params.append('height')
+            args.append(height)
+        if opacity is not None:
+            where_params.append('opacity')
+            args.append(opacity)
+        if background is not None:
+            where_params.append('background')
+            args.append(background)
+        if blur is not None:
+            where_params.append('blur')
+            args.append(blur)
+        if fallback is not None:
+            where_params.append('fallback')
+            args.append(fallback)
+
+        where = ''
+        if where_params:
+            where = 'WHERE ' + ' AND '.join([w + ' = ?' for w in where_params])
+
+        if order_by:
+            order_by = 'ORDER BY ' + order_by + ' DESC'
+
+        query = 'SELECT imgur_title, imgur_url FROM imgur_lookup ' \
+                'JOIN image_hash_lookup ON imgur_lookup.img_hash = image_hash_lookup.img_hash ' \
+                '%s %s' % (where, order_by)
+
+        try:
+            imgur_info = monitor_db.select(query, args=args)
+        except Exception as e:
+            logger.warn(u"Tautulli DataFactory :: Unable to execute database query for get_imgur_info: %s." % e)
+
+        return imgur_info
+
+    def set_imgur_info(self, img_hash=None, imgur_title=None, imgur_url=None, delete_hash=None):
+        monitor_db = database.MonitorDatabase()
+
+        keys = {'img_hash': img_hash}
+        values = {'imgur_title': imgur_title,
+                  'imgur_url': imgur_url,
+                  'delete_hash': delete_hash}
+
+        monitor_db.upsert('imgur_lookup', key_dict=keys, value_dict=values)
+
+    def delete_imgur_info(self, rating_key=None):
+        monitor_db = database.MonitorDatabase()
+
+        if rating_key:
+            query = 'SELECT imgur_title, delete_hash, fallback FROM imgur_lookup ' \
+                    'JOIN image_hash_lookup ON imgur_lookup.img_hash = image_hash_lookup.img_hash ' \
+                    'WHERE rating_key = ? '
+            args = [rating_key]
+            results = monitor_db.select(query, args=args)
+
+            for imgur_info in results:
+                if imgur_info['delete_hash']:
+                    helpers.delete_from_imgur(delete_hash=imgur_info['delete_hash'],
+                                              img_title=imgur_info['imgur_title'],
+                                              fallback=imgur_info['fallback'])
+
+            logger.info(u"Tautulli DataFactory :: Deleting Imgur info for rating_key %s from the database."
+                        % rating_key)
+            result = monitor_db.action('DELETE FROM imgur_lookup WHERE img_hash '
+                                       'IN (SELECT img_hash FROM image_hash_lookup WHERE rating_key = ?)',
+                                       [rating_key])
+
+            return True if result else False
+
+    def get_poster_info(self, rating_key='', metadata=None):
         poster_key = ''
         if str(rating_key).isdigit():
             poster_key = rating_key
@@ -1149,42 +1234,13 @@ class DataFactory(object):
         poster_info = {}
 
         if poster_key:
-            try:
-                query = 'SELECT poster_title, poster_url FROM poster_urls ' \
-                        'WHERE rating_key = ?'
-                poster_info = monitor_db.select_single(query, args=[poster_key])
-            except Exception as e:
-                logger.warn(u"Tautulli DataFactory :: Unable to execute database query for get_poster_url: %s." % e)
+            imgur_info = self.get_imgur_info(rating_key=poster_key, order_by='height', fallback='poster')
+            if imgur_info:
+                poster_info = {'poster_title': imgur_info[0]['imgur_title'],
+                               'poster_url': imgur_info[0]['imgur_url']}
 
         return poster_info
 
-    def set_poster_url(self, rating_key='', poster_title='', poster_url='', delete_hash=''):
-        monitor_db = database.MonitorDatabase()
-
-        if str(rating_key).isdigit():
-            keys = {'rating_key': int(rating_key)}
-
-            values = {'poster_title': poster_title,
-                      'poster_url': poster_url,
-                      'delete_hash': delete_hash}
-
-            monitor_db.upsert(table_name='poster_urls', key_dict=keys, value_dict=values)
-
-    def delete_poster_url(self, rating_key=''):
-        monitor_db = database.MonitorDatabase()
-
-        if rating_key:
-            poster_info = monitor_db.select_single('SELECT poster_title, delete_hash '
-                                                   'FROM poster_urls WHERE rating_key = ?',
-                                                   [rating_key])
-            if poster_info['delete_hash']:
-                helpers.delete_from_imgur(poster_info['delete_hash'], poster_info['poster_title'])
-
-            logger.info(u"Tautulli DataFactory :: Deleting poster_url for '%s' (rating_key %s) from the database."
-                        % (poster_info['poster_title'], rating_key))
-            result = monitor_db.action('DELETE FROM poster_urls WHERE rating_key = ?', [rating_key])
-            return True if result else False
-
     def get_lookup_info(self, rating_key='', metadata=None):
         monitor_db = database.MonitorDatabase()
 
@@ -1549,6 +1605,79 @@ class DataFactory(object):
             logger.warn(u"Tautulli DataFactory :: Unable to execute database query for delete_notification_log: %s." % e)
             return False
 
+    def get_newsletter_log(self, kwargs=None):
+        data_tables = datatables.DataTables()
+
+        columns = ['newsletter_log.id',
+                   'newsletter_log.timestamp',
+                   'newsletter_log.newsletter_id',
+                   'newsletter_log.agent_id',
+                   'newsletter_log.agent_name',
+                   'newsletter_log.notify_action',
+                   'newsletter_log.subject_text',
+                   'newsletter_log.body_text',
+                   'newsletter_log.start_date',
+                   'newsletter_log.end_date',
+                   'newsletter_log.uuid',
+                   'newsletter_log.success'
+                   ]
+        try:
+            query = data_tables.ssp_query(table_name='newsletter_log',
+                                          columns=columns,
+                                          custom_where=[],
+                                          group_by=[],
+                                          join_types=[],
+                                          join_tables=[],
+                                          join_evals=[],
+                                          kwargs=kwargs)
+        except Exception as e:
+            logger.warn(u"Tautulli DataFactory :: Unable to execute database query for get_newsletter_log: %s." % e)
+            return {'recordsFiltered': 0,
+                    'recordsTotal': 0,
+                    'draw': 0,
+                    'data': 'null',
+                    'error': 'Unable to execute database query.'}
+
+        newsletters = query['result']
+
+        rows = []
+        for item in newsletters:
+            row = {'id': item['id'],
+                   'timestamp': item['timestamp'],
+                   'newsletter_id': item['newsletter_id'],
+                   'agent_id': item['agent_id'],
+                   'agent_name': item['agent_name'],
+                   'notify_action': item['notify_action'],
+                   'subject_text': item['subject_text'],
+                   'body_text': item['body_text'],
+                   'start_date': item['start_date'],
+                   'end_date': item['end_date'],
+                   'uuid': item['uuid'],
+                   'success': item['success']
+                   }
+
+            rows.append(row)
+
+        dict = {'recordsFiltered': query['filteredCount'],
+                'recordsTotal': query['totalCount'],
+                'data': rows,
+                'draw': query['draw']
+                }
+
+        return dict
+
+    def delete_newsletter_log(self):
+        monitor_db = database.MonitorDatabase()
+
+        try:
+            logger.info(u"Tautulli DataFactory :: Clearing newsletter logs from database.")
+            monitor_db.action('DELETE FROM newsletter_log')
+            monitor_db.action('VACUUM')
+            return True
+        except Exception as e:
+            logger.warn(u"Tautulli DataFactory :: Unable to execute database query for delete_newsletter_log: %s." % e)
+            return False
+
     def get_user_devices(self, user_id=''):
         monitor_db = database.MonitorDatabase()
 

plexpy/helpers.py

@@ -20,6 +20,7 @@ import geoip2.database, geoip2.errors
 import gzip
 import hashlib
 import imghdr
+from itertools import izip_longest
 import ipwhois, ipwhois.exceptions, ipwhois.utils
 from IPy import IP
 import json
@@ -27,6 +28,7 @@ import math
 import maxminddb
 from operator import itemgetter
 import os
+from ratelimit import rate_limited
 import re
 import socket
 import sys
@@ -75,6 +77,7 @@ def addtoapi(*dargs, **dkwargs):
 
     return rd
 
+
 def multikeysort(items, columns):
     comparers = [((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns]
 
@@ -160,6 +163,7 @@ def convert_milliseconds(ms):
 
     return minutes
 
+
 def convert_milliseconds_to_minutes(ms):
 
     if str(ms).isdigit():
@@ -170,6 +174,7 @@ def convert_milliseconds_to_minutes(ms):
 
     return 0
 
+
 def convert_seconds(s):
 
     gmtime = time.gmtime(s)
@@ -180,6 +185,7 @@ def convert_seconds(s):
 
     return minutes
 
+
 def convert_seconds_to_minutes(s):
 
     if str(s).isdigit():
@@ -200,6 +206,7 @@ def now():
     now = datetime.datetime.now()
     return now.strftime("%Y-%m-%d %H:%M:%S")
 
+
 def human_duration(s, sig='dhms'):
 
     hd = ''
@@ -232,6 +239,7 @@ def human_duration(s, sig='dhms'):
 
     return hd
 
+
 def get_age(date):
 
     try:
@@ -384,6 +392,7 @@ def split_string(mystring, splitvar=','):
         mylist.append(each_word.strip())
     return mylist
 
+
 def create_https_certificates(ssl_cert, ssl_key):
     """
     Create a self-signed HTTPS certificate and store in it in
@@ -423,12 +432,14 @@ def cast_to_int(s):
     except (ValueError, TypeError):
         return 0
 
+
 def cast_to_float(s):
     try:
         return float(s)
     except (ValueError, TypeError):
         return 0
 
+
 def convert_xml_to_json(xml):
     o = xmltodict.parse(xml)
     return json.dumps(o)
@@ -451,12 +462,14 @@ def get_percent(value1, value2):
 
     return math.trunc(percent)
 
+
 def hex_to_int(hex):
     try:
         return int(hex, 16)
     except (ValueError, TypeError):
         return 0
 
+
 def parse_xml(unparsed=None):
     if unparsed:
         try:
@@ -472,10 +485,11 @@ def parse_xml(unparsed=None):
         logger.warn("XML parse request made but no data received.")
         return []
 
+
+def get_xml_attr(xml_key, attribute, return_bool=False, default_return=''):
     """
     Validate xml keys to make sure they exist and return their attribute value, return blank value is none found
     """
-def get_xml_attr(xml_key, attribute, return_bool=False, default_return=''):
     if xml_key.getAttribute(attribute):
         if return_bool:
             return True
@@ -487,6 +501,7 @@ def get_xml_attr(xml_key, attribute, return_bool=False, default_return=''):
         else:
             return default_return
 
+
 def process_json_kwargs(json_kwargs):
     params = {}
     if json_kwargs:
@@ -494,18 +509,21 @@ def process_json_kwargs(json_kwargs):
 
     return params
 
+
 def sanitize(string):
     if string:
         return unicode(string).replace('<','&lt;').replace('>','&gt;')
     else:
         return ''
 
+
 def is_public_ip(host):
     ip = is_valid_ip(get_ip(host))
     if ip and ip.iptype() == 'PUBLIC':
         return True
     return False
 
+
 def get_ip(host):
     ip_address = ''
     if is_valid_ip(host):
@@ -518,6 +536,7 @@ def get_ip(host):
             logger.error(u"IP Checker :: Bad IP or hostname provided.")
     return ip_address
 
+
 def is_valid_ip(address):
     try:
         return IP(address)
@@ -526,6 +545,7 @@ def is_valid_ip(address):
     except ValueError:
         return False
 
+
 def install_geoip_db():
     maxmind_url = 'http://geolite.maxmind.com/download/geoip/database/'
     geolite2_gz = 'GeoLite2-City.mmdb.gz'
@@ -586,6 +606,7 @@ def install_geoip_db():
 
     return True
 
+
 def uninstall_geoip_db():
     logger.debug(u"Tautulli Helpers :: Uninstalling the GeoLite2 database...")
     try:
@@ -599,6 +620,7 @@ def uninstall_geoip_db():
     logger.debug(u"Tautulli Helpers :: GeoLite2 database uninstalled successfully.")
     return True
 
+
 def geoip_lookup(ip_address):
     if not plexpy.CONFIG.GEOIP_DB:
         return 'GeoLite2 database not installed. Please install from the ' \
@@ -637,6 +659,7 @@ def geoip_lookup(ip_address):
 
     return geo_info
 
+
 def whois_lookup(ip_address):
 
     nets = []
@@ -673,6 +696,7 @@ def whois_lookup(ip_address):
 
     return whois_info
 
+
 # Taken from SickRage
 def anon_url(*url):
     """
@@ -680,7 +704,9 @@ def anon_url(*url):
     """
     return '' if None in url else '%s%s' % (plexpy.CONFIG.ANON_REDIRECT, ''.join(str(s) for s in url))
 
-def upload_to_imgur(imgPath, imgTitle=''):
+
+@rate_limited(450, 3600)
+def upload_to_imgur(img_data, img_title='', rating_key='', fallback=''):
     """ Uploads an image to Imgur """
     client_id = plexpy.CONFIG.IMGUR_CLIENT_ID
     img_url = delete_hash = ''
@@ -689,41 +715,33 @@ def upload_to_imgur(imgPath, imgTitle=''):
         logger.error(u"Tautulli Helpers :: Cannot upload poster to Imgur. No Imgur client id specified in the settings.")
         return img_url, delete_hash
 
-    try:
-        with open(imgPath, 'rb') as imgFile:
-            img = imgFile.read()
-    except IOError as e:
-        logger.error(u"Tautulli Helpers :: Unable to read image file for Imgur: %s" % e)
-        return img_url, delete_hash
-
     headers = {'Authorization': 'Client-ID %s' % client_id}
-    data = {'type': 'base64',
-            'image': base64.b64encode(img)}
-    if imgTitle:
-        data['title'] = imgTitle.encode('utf-8')
-        data['name'] = imgTitle.encode('utf-8') + '.jpg'
+    data = {'image': base64.b64encode(img_data),
+            'title': img_title.encode('utf-8'),
+            'name': str(rating_key) + '.png',
+            'type': 'png'}
 
     response, err_msg, req_msg = request.request_response2('https://api.imgur.com/3/image', 'POST',
                                                            headers=headers, data=data)
 
     if response and not err_msg:
-        t = '\'' + imgTitle + '\' ' if imgTitle else ''
-        logger.debug(u"Tautulli Helpers :: Image {}uploaded to Imgur.".format(t))
+        logger.debug(u"Tautulli Helpers :: Image '{}' ({}) uploaded to Imgur.".format(img_title, fallback))
         imgur_response_data = response.json().get('data')
         img_url = imgur_response_data.get('link', '').replace('http://', 'https://')
         delete_hash = imgur_response_data.get('deletehash', '')
     else:
         if err_msg:
-            logger.error(u"Tautulli Helpers :: Unable to upload image to Imgur: {}".format(err_msg))
+            logger.error(u"Tautulli Helpers :: Unable to upload image '{}' to Imgur: {}".format(img_title, err_msg))
         else:
-            logger.error(u"Tautulli Helpers :: Unable to upload image to Imgur.")
+            logger.error(u"Tautulli Helpers :: Unable to upload image '{}' to Imgur.".format(img_title))
 
         if req_msg:
             logger.debug(u"Tautulli Helpers :: Request response: {}".format(req_msg))
 
     return img_url, delete_hash
 
-def delete_from_imgur(delete_hash, imgTitle=''):
+
+def delete_from_imgur(delete_hash, img_title='', fallback=''):
     """ Deletes an image from Imgur """
     client_id = plexpy.CONFIG.IMGUR_CLIENT_ID
 
@@ -733,16 +751,16 @@ def delete_from_imgur(delete_hash, imgTitle=''):
                                                            headers=headers)
 
     if response and not err_msg:
-        t = '\'' + imgTitle + '\' ' if imgTitle else ''
-        logger.debug(u"Tautulli Helpers :: Image {}deleted from Imgur.".format(t))
+        logger.debug(u"Tautulli Helpers :: Image '{}' ({}) deleted from Imgur.".format(img_title, fallback))
         return True
     else:
         if err_msg:
-            logger.error(u"Tautulli Helpers :: Unable to delete image from Imgur: {}".format(err_msg))
+            logger.error(u"Tautulli Helpers :: Unable to delete image '{}' from Imgur: {}".format(img_title, err_msg))
         else:
-            logger.error(u"Tautulli Helpers :: Unable to delete image from Imgur.")
+            logger.error(u"Tautulli Helpers :: Unable to delete image '{}' from Imgur.".format(img_title))
         return False
 
+
 def cache_image(url, image=None):
     """
     Saves an image to the cache directory.
@@ -775,6 +793,7 @@ def cache_image(url, image=None):
 
     return imagefile, imagetype
 
+
 def build_datatables_json(kwargs, dt_columns, default_sort_col=None):
     """ Builds datatables json data
 
@@ -799,6 +818,7 @@ def build_datatables_json(kwargs, dt_columns, default_sort_col=None):
                  }
     return json.dumps(json_data)
 
+
 def humanFileSize(bytes, si=False):
     if str(bytes).isdigit():
         bytes = int(bytes)
@@ -822,6 +842,7 @@ def humanFileSize(bytes, si=False):
 
     return "{0:.1f} {1}".format(bytes, units[u])
 
+
 def parse_condition_logic_string(s, num_cond=0):
     """ Parse a logic string into a nested list
     Based on http://stackoverflow.com/a/23185606
@@ -906,6 +927,7 @@ def parse_condition_logic_string(s, num_cond=0):
 
     return stack.pop()
 
+
 def nested_list_to_string(l):
     for i, x in enumerate(l):
         if isinstance(x, list):
@@ -913,6 +935,7 @@ def nested_list_to_string(l):
     s = '(' + ' '.join(l) + ')'
     return s
 
+
 def eval_logic_groups_to_bool(logic_groups, eval_conds):
     first_cond = logic_groups[0]
 
@@ -934,6 +957,7 @@ def eval_logic_groups_to_bool(logic_groups, eval_conds):
 
     return result
 
+
 def get_plexpy_url(hostname=None):
     if plexpy.CONFIG.ENABLE_HTTPS:
         scheme = 'https'
@@ -966,3 +990,19 @@ def get_plexpy_url(hostname=None):
         root = ''
 
     return scheme + '://' + hostname + port + root
+
+
+def momentjs_to_arrow(format, duration=False):
+    invalid_formats = ['Mo', 'DDDo', 'do']
+    if duration:
+        invalid_formats += ['A', 'a']
+    for f in invalid_formats:
+        format = format.replace(f, '')
+    return format
+
+
+def grouper(iterable, n, fillvalue=None):
+    "Collect data into fixed-length chunks or blocks"
+    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
+    args = [iter(iterable)] * n
+    return izip_longest(fillvalue=fillvalue, *args)

plexpy/libraries.py

@@ -911,7 +911,7 @@ class Libraries(object):
         monitor_db = database.MonitorDatabase()
 
         try:
-            query = 'SELECT section_id, section_name FROM library_sections WHERE deleted_section = 0'
+            query = 'SELECT section_id, section_name, section_type FROM library_sections WHERE deleted_section = 0'
             result = monitor_db.select(query=query)
         except Exception as e:
             logger.warn(u"Tautulli Libraries :: Unable to execute database query for get_sections: %s." % e)
@@ -920,7 +920,8 @@ class Libraries(object):
         libraries = []
         for item in result:
             library = {'section_id': item['section_id'],
-                       'section_name': item['section_name']
+                       'section_name': item['section_name'],
+                       'section_type': item['section_type']
                        }
             libraries.append(library)
 

plexpy/log_reader.py

@@ -43,10 +43,9 @@ def get_log_tail(window=20, parsed=True, log_type="server"):
         clean_lines = []
         for i in log_lines:
             try:
-                i = helpers.latinToAscii(i)
                 log_time = i.split(' [')[0]
                 log_level = i.split('] ', 1)[1].split(' - ', 1)[0]
-                log_msg = i.split('] ', 1)[1].split(' - ',1)[1]
+                log_msg = unicode(i.split('] ', 1)[1].split(' - ', 1)[1], 'utf-8')
                 full_line = [log_time, log_level, log_msg]
                 clean_lines.append(full_line)
             except:

plexpy/newsletter_handler.py

@@ -0,0 +1,171 @@
+#  This file is part of Tautulli.
+#
+#  Tautulli is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  Tautulli is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Tautulli.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import time
+
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.cron import CronTrigger
+
+import plexpy
+import database
+import logger
+import newsletters
+
+
+NEWSLETTER_SCHED = BackgroundScheduler()
+
+
+def add_newsletter_each(newsletter_id=None, notify_action=None, **kwargs):
+    if not notify_action:
+        logger.debug(u"Tautulli NewsletterHandler :: Notify called but no action received.")
+        return
+
+    data = {'newsletter': True,
+            'newsletter_id': newsletter_id,
+            'notify_action': notify_action}
+    data.update(kwargs)
+    plexpy.NOTIFY_QUEUE.put(data)
+
+
+def schedule_newsletters(newsletter_id=None):
+    newsletters_list = newsletters.get_newsletters(newsletter_id=newsletter_id)
+
+    for newsletter in newsletters_list:
+        newsletter_job_name = '{} (newsletter_id {})'.format(newsletter['agent_label'], newsletter['id'])
+
+        if newsletter['active']:
+            schedule_newsletter_job('newsletter-{}'.format(newsletter['id']), name=newsletter_job_name,
+                                    func=add_newsletter_each, args=[newsletter['id'], 'on_cron'], cron=newsletter['cron'])
+        else:
+            schedule_newsletter_job('newsletter-{}'.format(newsletter['id']), name=newsletter_job_name,
+                                    remove_job=True)
+
+
+def schedule_newsletter_job(newsletter_job_id, name='', func=None, remove_job=False, args=None, cron=None):
+    if NEWSLETTER_SCHED.get_job(newsletter_job_id):
+        if remove_job:
+            NEWSLETTER_SCHED.remove_job(newsletter_job_id)
+            logger.info(u"Tautulli NewsletterHandler :: Removed scheduled newsletter: %s" % name)
+        else:
+            NEWSLETTER_SCHED.reschedule_job(
+                newsletter_job_id, args=args, trigger=CronTrigger().from_crontab(cron))
+            logger.info(u"Tautulli NewsletterHandler :: Re-scheduled newsletter: %s" % name)
+    elif not remove_job:
+        NEWSLETTER_SCHED.add_job(
+            func, args=args, id=newsletter_job_id, trigger=CronTrigger.from_crontab(cron))
+        logger.info(u"Tautulli NewsletterHandler :: Scheduled newsletter: %s" % name)
+
+
+def notify(newsletter_id=None, notify_action=None, **kwargs):
+    logger.info(u"Tautulli NewsletterHandler :: Preparing newsletter for newsletter_id %s." % newsletter_id)
+
+    newsletter_config = newsletters.get_newsletter_config(newsletter_id=newsletter_id)
+
+    if not newsletter_config:
+        return
+
+    if notify_action in ('test', 'api'):
+        subject = kwargs.pop('subject', None) or newsletter_config['subject']
+        body = kwargs.pop('body', None) or newsletter_config['body']
+        message = kwargs.pop('message', None) or newsletter_config['message']
+    else:
+        subject = newsletter_config['subject']
+        body = newsletter_config['body']
+        message = newsletter_config['message']
+
+    newsletter_agent = newsletters.get_agent_class(agent_id=newsletter_config['agent_id'],
+                                                   config=newsletter_config['config'],
+                                                   email_config=newsletter_config['email_config'],
+                                                   subject=subject,
+                                                   body=body,
+                                                   message=message
+                                                   )
+
+    # Set the newsletter state in the db
+    newsletter_log_id = set_notify_state(newsletter=newsletter_config,
+                                         notify_action=notify_action,
+                                         subject=newsletter_agent.subject_formatted,
+                                         body=newsletter_agent.body_formatted,
+                                         message=newsletter_agent.message_formatted,
+                                         start_date=newsletter_agent.start_date.format('YYYY-MM-DD'),
+                                         end_date=newsletter_agent.end_date.format('YYYY-MM-DD'),
+                                         newsletter_uuid=newsletter_agent.uuid)
+
+    # Send the notification
+    success = newsletter_agent.send()
+
+    if success:
+        set_notify_success(newsletter_log_id)
+        return True
+
+
+def set_notify_state(newsletter, notify_action, subject, body, message, start_date, end_date, newsletter_uuid):
+
+    if newsletter and notify_action:
+        db = database.MonitorDatabase()
+
+        keys = {'timestamp': int(time.time()),
+                'uuid': newsletter_uuid}
+
+        values = {'newsletter_id': newsletter['id'],
+                  'agent_id': newsletter['agent_id'],
+                  'agent_name': newsletter['agent_name'],
+                  'notify_action': notify_action,
+                  'subject_text': subject,
+                  'body_text': body,
+                  'message_text': message,
+                  'start_date': start_date,
+                  'end_date': end_date}
+
+        db.upsert(table_name='newsletter_log', key_dict=keys, value_dict=values)
+        return db.last_insert_id()
+    else:
+        logger.error(u"Tautulli NewsletterHandler :: Unable to set notify state.")
+
+
+def set_notify_success(newsletter_log_id):
+    keys = {'id': newsletter_log_id}
+    values = {'success': 1}
+
+    db = database.MonitorDatabase()
+    db.upsert(table_name='newsletter_log', key_dict=keys, value_dict=values)
+
+
+def get_newsletter(newsletter_uuid):
+    db = database.MonitorDatabase()
+    result = db.select_single('SELECT newsletter_id, start_date, end_date FROM newsletter_log '
+                              'WHERE uuid = ?', [newsletter_uuid])
+
+    if result:
+        newsletter_id = result['newsletter_id']
+        start_date = result['start_date']
+        end_date = result['end_date']
+
+        newsletter_file = 'newsletter_%s-%s_%s.html' % (start_date.replace('-', ''),
+                                                        end_date.replace('-', ''),
+                                                        newsletter_uuid)
+        newsletter_folder = plexpy.CONFIG.NEWSLETTER_DIR
+        newsletter_file_fp = os.path.join(newsletter_folder, newsletter_file)
+
+        if newsletter_file in os.listdir(newsletter_folder):
+            try:
+                with open(newsletter_file_fp, 'r') as n_file:
+                    newsletter = n_file.read()
+                return newsletter
+            except OSError as e:
+                logger.error(u"Tautulli NewsletterHandler :: Failed to retrieve newsletter '%s': %s" % (newsletter_uuid, e))
+        else:
+            logger.warn(u"Tautulli NewsletterHandler :: Newsletter '%s' file is missing." % newsletter_uuid)

plexpy/newsletters.py

@@ -0,0 +1,787 @@
+#  This file is part of Tautulli.
+#
+#  Tautulli is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  Tautulli is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Tautulli.  If not, see <http://www.gnu.org/licenses/>.
+
+import arrow
+import json
+from itertools import groupby
+from mako.lookup import TemplateLookup
+from mako import exceptions
+import os
+
+import plexpy
+import common
+import database
+import helpers
+import libraries
+import logger
+import newsletter_handler
+import pmsconnect
+from notifiers import send_notification, EMAIL
+
+
+AGENT_IDS = {
+    'recently_added': 0
+}
+
+
+def available_newsletter_agents():
+    agents = [
+        {
+            'label': 'Recently Added',
+            'name': 'recently_added',
+            'id': AGENT_IDS['recently_added']
+        }
+    ]
+
+    return agents
+
+
+def available_notification_actions():
+    actions = [{'label': 'Schedule',
+                'name': 'on_cron',
+                'description': 'Trigger a notification on a certain schedule.',
+                'subject': 'Tautulli Newsletter',
+                'body': 'Tautulli Newsletter',
+                'message': '',
+                'icon': 'fa-calendar',
+                'media_types': ('newsletter',)
+                }
+               ]
+
+    return actions
+
+
+def get_agent_class(agent_id=None, config=None, email_config=None, start_date=None, end_date=None,
+                    subject=None, body=None, message=None):
+    if str(agent_id).isdigit():
+        agent_id = int(agent_id)
+
+        kwargs = {'config': config,
+                  'email_config': email_config,
+                  'start_date': start_date,
+                  'end_date': end_date,
+                  'subject': subject,
+                  'body': body,
+                  'message': message}
+
+        if agent_id == 0:
+            return RecentlyAdded(**kwargs)
+        else:
+            return Newsletter(**kwargs)
+    else:
+        return None
+
+
+def get_newsletter_agents():
+    return tuple(a['name'] for a in sorted(available_newsletter_agents(), key=lambda k: k['label']))
+
+
+def get_newsletters(newsletter_id=None):
+    where = where_id = ''
+    args = []
+
+    if newsletter_id:
+        where = 'WHERE '
+        if newsletter_id:
+            where_id += 'id = ?'
+            args.append(newsletter_id)
+        where += ' AND '.join([w for w in [where_id] if w])
+
+    db = database.MonitorDatabase()
+    result = db.select('SELECT id, agent_id, agent_name, agent_label, '
+                       'friendly_name, cron, active FROM newsletters %s' % where, args=args)
+
+    return result
+
+
+def delete_newsletter(newsletter_id=None):
+    db = database.MonitorDatabase()
+
+    if str(newsletter_id).isdigit():
+        logger.debug(u"Tautulli Newsletters :: Deleting newsletter_id %s from the database."
+                     % newsletter_id)
+        result = db.action('DELETE FROM newsletters WHERE id = ?', args=[newsletter_id])
+        return True
+    else:
+        return False
+
+
+def get_newsletter_config(newsletter_id=None):
+    if str(newsletter_id).isdigit():
+        newsletter_id = int(newsletter_id)
+    else:
+        logger.error(u"Tautulli Newsletters :: Unable to retrieve newsletter config: invalid newsletter_id %s."
+                     % newsletter_id)
+        return None
+
+    db = database.MonitorDatabase()
+    result = db.select_single('SELECT * FROM newsletters WHERE id = ?', args=[newsletter_id])
+
+    if not result:
+        return None
+
+    try:
+        config = json.loads(result.pop('newsletter_config', '{}'))
+        email_config = json.loads(result.pop('email_config', '{}'))
+        subject = result.pop('subject')
+        body = result.pop('body')
+        message = result.pop('message')
+        newsletter_agent = get_agent_class(agent_id=result['agent_id'], config=config, email_config=email_config,
+                                           subject=subject, body=body, message=message)
+    except Exception as e:
+        logger.error(u"Tautulli Newsletters :: Failed to get newsletter config options: %s." % e)
+        return
+
+    result['subject'] = newsletter_agent.subject
+    result['body'] = newsletter_agent.body
+    result['message'] = newsletter_agent.message
+    result['config'] = newsletter_agent.config
+    result['email_config'] = newsletter_agent.email_config
+    result['config_options'] = newsletter_agent.return_config_options()
+    result['email_config_options'] = newsletter_agent.return_email_config_options()
+
+    return result
+
+
+def add_newsletter_config(agent_id=None, **kwargs):
+    if str(agent_id).isdigit():
+        agent_id = int(agent_id)
+    else:
+        logger.error(u"Tautulli Newsletters :: Unable to add new newsletter: invalid agent_id %s."
+                     % agent_id)
+        return False
+
+    agent = next((a for a in available_newsletter_agents() if a['id'] == agent_id), None)
+
+    if not agent:
+        logger.error(u"Tautulli Newsletters :: Unable to retrieve new newsletter agent: invalid agent_id %s."
+                     % agent_id)
+        return False
+
+    agent_class = get_agent_class(agent_id=agent['id'])
+
+    keys = {'id': None}
+    values = {'agent_id': agent['id'],
+              'agent_name': agent['name'],
+              'agent_label': agent['label'],
+              'friendly_name': '',
+              'newsletter_config': json.dumps(agent_class.config),
+              'email_config': json.dumps(agent_class.email_config),
+              'subject': agent_class.subject,
+              'body': agent_class.body,
+              'message': agent_class.message
+              }
+
+    db = database.MonitorDatabase()
+    try:
+        db.upsert(table_name='newsletters', key_dict=keys, value_dict=values)
+        newsletter_id = db.last_insert_id()
+        logger.info(u"Tautulli Newsletters :: Added new newsletter agent: %s (newsletter_id %s)."
+                    % (agent['label'], newsletter_id))
+        return newsletter_id
+    except Exception as e:
+        logger.warn(u"Tautulli Newsletters :: Unable to add newsletter agent: %s." % e)
+        return False
+
+
+def set_newsletter_config(newsletter_id=None, agent_id=None, **kwargs):
+    if str(agent_id).isdigit():
+        agent_id = int(agent_id)
+    else:
+        logger.error(u"Tautulli Newsletters :: Unable to set exisiting newsletter: invalid agent_id %s."
+                     % agent_id)
+        return False
+
+    agent = next((a for a in available_newsletter_agents() if a['id'] == agent_id), None)
+
+    if not agent:
+        logger.error(u"Tautulli Newsletters :: Unable to retrieve existing newsletter agent: invalid agent_id %s."
+                     % agent_id)
+        return False
+
+    config_prefix = 'newsletter_config_'
+    email_config_prefix = 'newsletter_email_'
+
+    newsletter_config = {k[len(config_prefix):]: kwargs.pop(k)
+                         for k in kwargs.keys() if k.startswith(config_prefix)}
+    email_config = {k[len(email_config_prefix):]: kwargs.pop(k)
+                    for k in kwargs.keys() if k.startswith(email_config_prefix)}
+
+    subject = kwargs.pop('subject')
+    body = kwargs.pop('body')
+    message = kwargs.pop('message')
+
+    agent_class = get_agent_class(agent_id=agent['id'], config=newsletter_config, email_config=email_config,
+                                  subject=subject, body=body, message=message)
+
+    keys = {'id': newsletter_id}
+    values = {'agent_id': agent['id'],
+              'agent_name': agent['name'],
+              'agent_label': agent['label'],
+              'friendly_name': kwargs.get('friendly_name', ''),
+              'newsletter_config': json.dumps(agent_class.config),
+              'email_config': json.dumps(agent_class.email_config),
+              'subject': agent_class.subject,
+              'body': agent_class.body,
+              'message': agent_class.message,
+              'cron': kwargs.get('cron'),
+              'active': kwargs.get('active')
+              }
+
+    db = database.MonitorDatabase()
+    try:
+        db.upsert(table_name='newsletters', key_dict=keys, value_dict=values)
+        logger.info(u"Tautulli Newsletters :: Updated newsletter agent: %s (newsletter_id %s)."
+                    % (agent['label'], newsletter_id))
+        newsletter_handler.schedule_newsletters(newsletter_id=newsletter_id)
+        return True
+    except Exception as e:
+        logger.warn(u"Tautulli Newsletters :: Unable to update newsletter agent: %s." % e)
+        return False
+
+
+def send_newsletter(newsletter_id=None, subject=None, body=None, message=None, newsletter_log_id=None, **kwargs):
+    newsletter_config = get_newsletter_config(newsletter_id=newsletter_id)
+    if newsletter_config:
+        agent = get_agent_class(agent_id=newsletter_config['agent_id'],
+                                config=newsletter_config['config'],
+                                email_config=newsletter_config['email_config'],
+                                subject=subject,
+                                body=body,
+                                messsage=message)
+        return agent.send()
+    else:
+        logger.debug(u"Tautulli Newsletters :: Notification requested but no newsletter_id received.")
+
+
+def serve_template(templatename, **kwargs):
+    interface_dir = os.path.join(str(plexpy.PROG_DIR), 'data/interfaces/')
+    template_dir = os.path.join(str(interface_dir), plexpy.CONFIG.NEWSLETTER_TEMPLATES)
+
+    _hplookup = TemplateLookup(directories=[template_dir], default_filters=['unicode', 'h'])
+
+    try:
+        template = _hplookup.get_template(templatename)
+        return template.render(**kwargs)
+    except:
+        return exceptions.html_error_template().render()
+
+
+def generate_newsletter_uuid():
+    uuid = ''
+    uuid_exists = 0
+    db = database.MonitorDatabase()
+
+    while not uuid or uuid_exists:
+        uuid = plexpy.generate_uuid()[:8]
+        result = db.select_single(
+            'SELECT EXISTS(SELECT uuid FROM newsletter_log WHERE uuid = ?) as uuid_exists', [uuid])
+        uuid_exists = result['uuid_exists']
+
+    return uuid
+
+
+class Newsletter(object):
+    NAME = ''
+    _DEFAULT_CONFIG = {'custom_cron': 0,
+                       'last_days': 7,
+                       'formatted': 1,
+                       'notifier_id': 0}
+    _DEFAULT_EMAIL_CONFIG = EMAIL().return_default_config()
+    _DEFAULT_EMAIL_CONFIG['from_name'] = 'Tautulli Newsletter'
+    _DEFAULT_EMAIL_CONFIG['notifier_id'] = 0
+    _DEFAULT_SUBJECT = 'Tautulli Newsletter'
+    _DEFAULT_BODY = 'View the newsletter here: {newsletter_url}'
+    _DEFAULT_MESSAGE = ''
+    _TEMPLATE_MASTER = ''
+    _TEMPLATE = ''
+
+    def __init__(self, config=None, email_config=None, start_date=None, end_date=None,
+                 subject=None, body=None, message=None):
+        self.config = self.set_config(config=config, default=self._DEFAULT_CONFIG)
+        self.email_config = self.set_config(config=email_config, default=self._DEFAULT_EMAIL_CONFIG)
+        self.uuid = generate_newsletter_uuid()
+
+        self.start_date = None
+        self.end_date = None
+
+        if end_date:
+            try:
+                self.end_date = arrow.get(end_date, 'YYYY-MM-DD', tzinfo='local').ceil('day')
+            except ValueError:
+                pass
+
+        if self.end_date is None:
+            self.end_date = arrow.now().ceil('day')
+
+        if start_date:
+            try:
+                self.start_date = arrow.get(start_date, 'YYYY-MM-DD', tzinfo='local').floor('day')
+            except ValueError:
+                pass
+
+        if self.start_date is None:
+            self.start_date = self.end_date.shift(days=-self.config['last_days']+1).floor('day')
+
+        self.end_time = self.end_date.timestamp
+        self.start_time = self.start_date.timestamp
+
+        self.parameters = self.build_params()
+        self.subject = subject or self._DEFAULT_SUBJECT
+        self.body = body or self._DEFAULT_BODY
+        self.message = message or self._DEFAULT_MESSAGE
+        self.subject_formatted, self.body_formatted, self.message_formatted = self.build_text()
+
+        self.data = {}
+        self.newsletter = None
+
+        self.is_preview = False
+
+    def set_config(self, config=None, default=None):
+        return self._validate_config(config=config, default=default)
+
+    def _validate_config(self, config=None, default=None):
+        if config is None:
+            return default
+
+        new_config = {}
+        for k, v in default.iteritems():
+            if isinstance(v, int):
+                new_config[k] = helpers.cast_to_int(config.get(k, v))
+            elif isinstance(v, list):
+                c = config.get(k, v)
+                if not isinstance(c, list):
+                    new_config[k] = [c]
+                else:
+                    new_config[k] = c
+            else:
+                new_config[k] = config.get(k, v)
+
+        return new_config
+
+    def retrieve_data(self):
+        pass
+
+    def _has_data(self):
+        return False
+
+    def raw_data(self, preview=False):
+        if preview:
+            self.is_preview = True
+
+        self.retrieve_data()
+
+        return {'title': self.NAME,
+                'parameters': self.parameters,
+                'data': self.data}
+
+    def generate_newsletter(self, preview=False, master=False):
+        if preview:
+            self.is_preview = True
+
+        if master:
+            template = self._TEMPLATE_MASTER
+        else:
+            template = self._TEMPLATE
+
+        self.retrieve_data()
+
+        return serve_template(
+            templatename=template,
+            uuid=self.uuid,
+            subject=self.subject_formatted,
+            body=self.body_formatted,
+            message=self.message_formatted,
+            parameters=self.parameters,
+            data=self.data,
+            preview=self.is_preview
+        )
+
+    def send(self):
+        self.newsletter = self.generate_newsletter()
+
+        self._save()
+        return self._send()
+
+    def _save(self):
+        newsletter_file = 'newsletter_%s-%s_%s.html' % (self.start_date.format('YYYYMMDD'),
+                                                        self.end_date.format('YYYYMMDD'),
+                                                        self.uuid)
+        newsletter_folder = plexpy.CONFIG.NEWSLETTER_DIR
+        newsletter_file_fp = os.path.join(newsletter_folder, newsletter_file)
+
+        # In case the user has deleted it manually
+        if not os.path.exists(newsletter_folder):
+            os.makedirs(newsletter_folder)
+
+        try:
+            with open(newsletter_file_fp, 'wb') as n_file:
+                for line in self.newsletter.encode('utf-8').splitlines():
+                    if '<!-- IGNORE SAVE -->' not in line:
+                        n_file.write(line + '\r\n')
+
+            logger.info(u"Tautulli Newsletters :: %s newsletter saved to %s" % (self.NAME, newsletter_file))
+        except OSError as e:
+            logger.error(u"Tautulli Newsletters :: Failed to save %s newsletter to %s: %s"
+                         % (self.NAME, newsletter_file, e))
+
+    def _send(self):
+        if not self._has_data():
+            logger.warn(u"Tautulli Newsletters :: %s newsletter has no data. Newsletter not sent." % self.NAME)
+            return False
+
+        if self.config['formatted']:
+            if self.email_config['notifier_id']:
+                return send_notification(
+                    notifier_id=self.email_config['notifier_id'],
+                    subject=self.subject_formatted,
+                    body=self.newsletter
+                )
+
+            else:
+                email = EMAIL(config=self.email_config)
+                return email.notify(
+                    subject=self.subject_formatted,
+                    body=self.newsletter
+                )
+        elif self.config['notifier_id']:
+            return send_notification(
+                    notifier_id=self.config['notifier_id'],
+                    subject=self.subject_formatted,
+                    body=self.body_formatted
+                )
+
+    def build_params(self):
+        parameters = self._build_params()
+
+        return parameters
+
+    def _build_params(self):
+        date_format = helpers.momentjs_to_arrow(plexpy.CONFIG.DATE_FORMAT)
+
+        base_url = plexpy.CONFIG.HTTP_BASE_URL or helpers.get_plexpy_url()
+
+        parameters = {
+            'server_name': plexpy.CONFIG.PMS_NAME,
+            'start_date': self.start_date.format(date_format),
+            'end_date': self.end_date.format(date_format),
+            'newsletter_days': self.config['last_days'],
+            'newsletter_url': base_url.rstrip('/') + plexpy.HTTP_ROOT + 'newsletter/' + self.uuid,
+            'newsletter_uuid': self.uuid
+        }
+
+        return parameters
+
+    def build_text(self):
+        from notification_handler import CustomFormatter
+        custom_formatter = CustomFormatter()
+
+        try:
+            subject = custom_formatter.format(unicode(self.subject), **self.parameters)
+        except LookupError as e:
+            logger.error(
+                u"Tautulli Newsletter :: Unable to parse parameter %s in newsletter subject. Using fallback." % e)
+            subject = unicode(self._DEFAULT_SUBJECT).format(**self.parameters)
+        except Exception as e:
+            logger.error(
+                u"Tautulli Newsletter :: Unable to parse custom newsletter subject: %s. Using fallback." % e)
+            subject = unicode(self._DEFAULT_SUBJECT).format(**self.parameters)
+
+        try:
+            body = custom_formatter.format(unicode(self.body), **self.parameters)
+        except LookupError as e:
+            logger.error(
+                u"Tautulli Newsletter :: Unable to parse parameter %s in newsletter body. Using fallback." % e)
+            body = unicode(self._DEFAULT_BODY).format(**self.parameters)
+        except Exception as e:
+            logger.error(
+                u"Tautulli Newsletter :: Unable to parse custom newsletter body: %s. Using fallback." % e)
+            body = unicode(self._DEFAULT_BODY).format(**self.parameters)
+
+        try:
+            message = custom_formatter.format(unicode(self.message), **self.parameters)
+        except LookupError as e:
+            logger.error(
+                u"Tautulli Newsletter :: Unable to parse parameter %s in newsletter message. Using fallback." % e)
+            message = unicode(self._DEFAULT_MESSAGE).format(**self.parameters)
+        except Exception as e:
+            logger.error(
+                u"Tautulli Newsletter :: Unable to parse custom newsletter message: %s. Using fallback." % e)
+            message = unicode(self._DEFAULT_MESSAGE).format(**self.parameters)
+
+        return subject, body, message
+
+    def return_config_options(self):
+        return self._return_config_options()
+
+    def _return_config_options(self):
+        config_options = [
+            {'label': 'Number of Days',
+             'value': self.config['last_days'],
+             'name': 'newsletter_config_last_days',
+             'description': 'The past number of days to include in the newsletter.',
+             'input_type': 'number'
+             }
+        ]
+
+        return config_options
+
+    def return_email_config_options(self):
+        config_options = EMAIL(self.email_config).return_config_options()
+        for c in config_options:
+            c['name'] = 'newsletter_' + c['name']
+        return config_options
+
+
+class RecentlyAdded(Newsletter):
+    """
+    Recently Added Newsletter
+    """
+    NAME = 'Recently Added'
+    _DEFAULT_CONFIG = Newsletter._DEFAULT_CONFIG.copy()
+    _DEFAULT_CONFIG['incl_libraries'] = []
+    _DEFAULT_SUBJECT = 'Recently Added to {server_name}! ({end_date})'
+    _DEFAULT_BODY = 'View the newsletter here: {newsletter_url}'
+    _DEFAULT_MESSAGE = ''
+    _TEMPLATE_MASTER = 'recently_added_master.html'
+    _TEMPLATE = 'recently_added.html'
+
+    def _get_recently_added(self, media_type=None):
+        from notification_handler import format_group_index
+
+        pms_connect = pmsconnect.PmsConnect()
+
+        recently_added = []
+        done = False
+        start = 0
+
+        while not done:
+            recent_items = pms_connect.get_recently_added_details(start=str(start), count='10', type=media_type)
+            filtered_items = [i for i in recent_items['recently_added']
+                              if self.start_time < helpers.cast_to_int(i['added_at']) < self.end_time]
+            if len(filtered_items) < 10:
+                done = True
+            else:
+                start += 10
+
+            recently_added.extend(filtered_items)
+
+        if media_type == 'movie':
+            movie_list = []
+            for item in recently_added:
+                # Filter included libraries
+                if item['section_id'] not in self.config['incl_libraries']:
+                    continue
+
+                movie_list.append(item)
+
+            recently_added = movie_list
+
+        if media_type == 'show':
+            shows_list = []
+            show_rating_keys = []
+            for item in recently_added:
+                # Filter included libraries
+                if item['section_id'] not in self.config['incl_libraries']:
+                    continue
+
+                if item['media_type'] == 'show':
+                    show_rating_key = item['rating_key']
+                elif item['media_type'] == 'season':
+                    show_rating_key = item['parent_rating_key']
+                elif item['media_type'] == 'episode':
+                    show_rating_key = item['grandparent_rating_key']
+
+                if show_rating_key in show_rating_keys:
+                    continue
+
+                show_metadata = pms_connect.get_metadata_details(show_rating_key, media_info=False)
+                children = pms_connect.get_item_children(show_rating_key, get_grandchildren=True)
+                filtered_children = [i for i in children['children_list']
+                                     if self.start_time < helpers.cast_to_int(i['added_at']) < self.end_time]
+                filtered_children.sort(key=lambda x: int(x['parent_media_index']))
+
+                seasons = []
+                for k, v in groupby(filtered_children, key=lambda x: x['parent_media_index']):
+                    episodes = list(v)
+                    num, num00 = format_group_index([helpers.cast_to_int(d['media_index']) for d in episodes])
+
+                    seasons.append({'media_index': k,
+                                    'episode_range': num00,
+                                    'episode_count': len(episodes),
+                                    'episode': episodes})
+
+                num, num00 = format_group_index([helpers.cast_to_int(d['media_index']) for d in seasons])
+
+                show_metadata['season_range'] = num00
+                show_metadata['season_count'] = len(seasons)
+                show_metadata['season'] = seasons
+
+                shows_list.append(show_metadata)
+                show_rating_keys.append(show_rating_key)
+
+            recently_added = shows_list
+
+        if media_type == 'artist':
+            artists_list = []
+            artist_rating_keys = []
+            for item in recently_added:
+                # Filter included libraries
+                if item['section_id'] not in self.config['incl_libraries']:
+                    continue
+
+                if item['media_type'] == 'artist':
+                    artist_rating_key = item['rating_key']
+                elif item['media_type'] == 'album':
+                    artist_rating_key = item['parent_rating_key']
+                elif item['media_type'] == 'track':
+                    artist_rating_key = item['grandparent_rating_key']
+
+                if artist_rating_key in artist_rating_keys:
+                    continue
+
+                artist_metadata = pms_connect.get_metadata_details(artist_rating_key, media_info=False)
+                children = pms_connect.get_item_children(artist_rating_key)
+                filtered_children = [i for i in children['children_list']
+                                     if self.start_time < helpers.cast_to_int(i['added_at']) < self.end_time]
+                filtered_children.sort(key=lambda x: x['added_at'])
+
+                albums = []
+                for a in filtered_children:
+                    album_metadata = pms_connect.get_metadata_details(a['rating_key'], media_info=False)
+                    album_metadata['track_count'] = helpers.cast_to_int(album_metadata['children_count'])
+                    albums.append(album_metadata)
+
+                artist_metadata['album_count'] = len(albums)
+                artist_metadata['album'] = albums
+
+                artists_list.append(artist_metadata)
+                artist_rating_keys.append(artist_rating_key)
+
+            recently_added = artists_list
+
+        return recently_added
+
+    def retrieve_data(self):
+        from notification_handler import get_imgur_info, set_hash_image_info
+
+        if not self.config['incl_libraries']:
+            logger.warn(u"Tautulli Newsletters :: Failed to retrieve %s newsletter data: no libraries selected." % self.NAME)
+
+        media_types = {s['section_type'] for s in self._get_sections()
+                       if str(s['section_id']) in self.config['incl_libraries']}
+
+        recently_added = {}
+        for media_type in media_types:
+            if media_type not in recently_added:
+                recently_added[media_type] = self._get_recently_added(media_type)
+
+        movies = recently_added.get('movie', [])
+        shows = recently_added.get('show', [])
+        artists = recently_added.get('artist', [])
+        albums = [a for artist in artists for a in artist['album']]
+
+        if self.is_preview or plexpy.CONFIG.NEWSLETTER_SELF_HOSTED:
+            for item in movies + shows + albums:
+                item['thumb_hash'] = set_hash_image_info(
+                    img=item['thumb'], width=150, height=225, fallback='poster')
+
+                if item['art']:
+                    item['art_hash'] = set_hash_image_info(
+                        img=item['art'], width=500, height=280,
+                        opacity=25, background='282828', blur=3, fallback='art')
+                else:
+                    item['art_hash'] = ''
+
+                item['poster_url'] = ''
+                item['art_url'] = ''
+
+        else:
+            # Upload posters and art to Imgur
+            for item in movies + shows + albums:
+                imgur_info = get_imgur_info(
+                    img=item['thumb'], rating_key=item['rating_key'], title=item['title'],
+                    width=150, height=225, fallback='poster')
+
+                item['poster_url'] = imgur_info.get('imgur_url') or common.ONLINE_POSTER_THUMB
+
+                imgur_info = get_imgur_info(
+                    img=item['art'], rating_key=item['rating_key'], title=item['title'],
+                    width=500, height=280, opacity=25, background='282828', blur=3, fallback='art')
+
+                item['art_url'] = imgur_info.get('imgur_url')
+
+                item['thumb_hash'] = ''
+                item['art_hash'] = ''
+
+        self.data['recently_added'] = recently_added
+
+        return self.data
+
+    def _has_data(self):
+        recently_added = self.data.get('recently_added')
+        if recently_added and \
+                recently_added.get('movie') or \
+                recently_added.get('show') or \
+                recently_added.get('artist'):
+            return True
+
+        return False
+
+    def _get_sections(self):
+        return libraries.Libraries().get_sections()
+
+    def _get_sections_options(self):
+        library_types = {'movie': 'Movie Libraries',
+                         'show': 'TV Show Libraries',
+                         'artist': 'Music Libraries'}
+        sections = {}
+        for s in self._get_sections():
+            if s['section_type'] != 'photo':
+                library_type = library_types[s['section_type']]
+                group = sections.get(library_type, [])
+                group.append({'value': s['section_id'],
+                              'text': s['section_name']})
+                sections[library_type] = group
+        return sections
+
+    def build_params(self):
+        parameters = self._build_params()
+
+        newsletter_libraries = []
+        for s in self._get_sections():
+            if str(s['section_id']) in self.config['incl_libraries']:
+                newsletter_libraries.append(s['section_name'])
+
+        parameters['newsletter_libraries'] = ', '.join(sorted(newsletter_libraries))
+        parameters['pms_identifier'] = plexpy.CONFIG.PMS_IDENTIFIER
+        parameters['pms_web_url'] = plexpy.CONFIG.PMS_WEB_URL
+
+        return parameters
+
+    def return_config_options(self):
+        config_options = self._return_config_options()
+
+        additional_config = [
+            {'label': 'Included Libraries',
+             'value': self.config['incl_libraries'],
+             'description': 'Select the libraries to include in the newsletter.',
+             'name': 'newsletter_config_incl_libraries',
+             'input_type': 'selectize',
+             'select_options': self._get_sections_options()
+             }
+        ]
+
+        return config_options + additional_config

plexpy/notification_handler.py

@@ -17,6 +17,7 @@
 import arrow
 import bleach
 from collections import Counter, defaultdict
+import hashlib
 from itertools import groupby
 import json
 from operator import itemgetter
@@ -39,6 +40,7 @@ import plextv
 import pmsconnect
 import request
 import users
+from newsletter_handler import notify as notify_newsletter
 
 
 def process_queue():
@@ -50,7 +52,9 @@ def process_queue():
             break
         elif params:
             try:
-                if 'notify' in params:
+                if 'newsletter' in params:
+                    notify_newsletter(**params)
+                elif 'notification' in params:
                     notify(**params)
                 else:
                     add_notifier_each(**params)
@@ -111,7 +115,7 @@ def add_notifier_each(notifier_id=None, notify_action=None, stream_data=None, ti
             # Check custom user conditions
             if manual_trigger or notify_custom_conditions(notifier_id=notifier['id'], parameters=parameters):
                 # Add each notifier to the queue
-                data = {'notify': True,
+                data = {'notification': True,
                         'notifier_id': notifier['id'],
                         'notify_action': notify_action,
                         'stream_data': stream_data,
@@ -448,9 +452,9 @@ def set_notify_success(notification_id):
 
 def build_media_notify_params(notify_action=None, session=None, timeline=None, manual_trigger=False, **kwargs):
     # Get time formats
-    date_format = plexpy.CONFIG.DATE_FORMAT.replace('Do','')
-    time_format = plexpy.CONFIG.TIME_FORMAT.replace('Do','')
-    duration_format = plexpy.CONFIG.TIME_FORMAT.replace('Do','').replace('a','').replace('A','')
+    date_format = helpers.momentjs_to_arrow(plexpy.CONFIG.DATE_FORMAT)
+    time_format = helpers.momentjs_to_arrow(plexpy.CONFIG.TIME_FORMAT)
+    duration_format = helpers.momentjs_to_arrow(plexpy.CONFIG.TIME_FORMAT, duration=True)
 
     # Get metadata for the item
     if session:
@@ -628,8 +632,14 @@ def build_media_notify_params(notify_action=None, session=None, timeline=None, m
     else:
         poster_thumb = ''
 
-    if plexpy.CONFIG.NOTIFY_UPLOAD_POSTERS:
-        poster_info = get_poster_info(poster_thumb=poster_thumb, poster_key=poster_key, poster_title=poster_title)
+    if plexpy.CONFIG.NOTIFY_UPLOAD_POSTERS == 1:
+        imgur_info = get_imgur_info(img=poster_thumb, rating_key=poster_key, title=poster_title, fallback='poster')
+        poster_info = {'poster_title': imgur_info['imgur_title'], 'poster_url': imgur_info['imgur_url']}
+        notify_params.update(poster_info)
+    elif plexpy.CONFIG.NOTIFY_UPLOAD_POSTERS == 2 and plexpy.CONFIG.HTTP_BASE_URL:
+        img_hash = set_hash_image_info(img=poster_thumb, fallback='poster')
+        poster_info = {'poster_title': poster_title,
+                       'poster_url': plexpy.CONFIG.HTTP_BASE_URL + plexpy.HTTP_ROOT + 'image/' + img_hash}
         notify_params.update(poster_info)
 
     if ((manual_trigger or plexpy.CONFIG.NOTIFY_GROUP_RECENTLY_ADDED_GRANDPARENT)
@@ -1065,45 +1075,88 @@ def format_group_index(group_keys):
     return ','.join(num) or '0', ','.join(num00) or '00'
 
 
-def get_poster_info(poster_thumb, poster_key, poster_title):
+def get_imgur_info(img=None, rating_key=None, title='', width=600, height=1000,
+                   opacity=100, background='000000', blur=0, fallback=None):
+    imgur_info = {'imgur_title': '', 'imgur_url': ''}
+
+    image_info = {'img': img,
+                  'rating_key': rating_key,
+                  'width': width,
+                  'height': height,
+                  'opacity': opacity,
+                  'background': background,
+                  'blur': blur,
+                  'fallback': fallback}
+
     # Try to retrieve poster info from the database
     data_factory = datafactory.DataFactory()
-    poster_info = data_factory.get_poster_info(rating_key=poster_key)
+    database_imgur_info = data_factory.get_imgur_info(**image_info)
 
-    # If no previous poster info
-    if not poster_info and poster_thumb:
-        try:
-            thread_name = str(threading.current_thread().ident)
-            poster_file = os.path.join(plexpy.CONFIG.CACHE_DIR, 'cache-poster-%s' % thread_name)
+    if database_imgur_info:
+        imgur_info = database_imgur_info[0]
 
-            # Retrieve the poster from Plex and cache to file
+    elif not database_imgur_info and img:
         pms_connect = pmsconnect.PmsConnect()
-            result = pms_connect.get_image(img=poster_thumb)
+        result = pms_connect.get_image(**image_info)
+
         if result and result[0]:
-                with open(poster_file, 'wb') as f:
-                    f.write(result[0])
-            else:
-                raise Exception(u'PMS image request failed')
+            imgur_url, delete_hash = helpers.upload_to_imgur(img_data=result[0],
+                                                             img_title=title,
+                                                             rating_key=rating_key,
+                                                             fallback=fallback)
 
-            # Upload poster_thumb to Imgur and get link
-            poster_url, delete_hash = helpers.upload_to_imgur(poster_file, poster_title)
-
-            if poster_url:
-                # Create poster info
-                poster_info = {'poster_title': poster_title, 'poster_url': poster_url}
-
-                # Save the poster url in the database
-                data_factory.set_poster_url(rating_key=poster_key,
-                                            poster_title=poster_title,
-                                            poster_url=poster_url,
+            if imgur_url:
+                img_hash = set_hash_image_info(**image_info)
+                data_factory.set_imgur_info(img_hash=img_hash,
+                                            imgur_title=title,
+                                            imgur_url=imgur_url,
                                             delete_hash=delete_hash)
 
-            # Delete the cached poster
-            os.remove(poster_file)
-        except Exception as e:
-            logger.error(u"Tautulli NotificationHandler :: Unable to retrieve poster for rating_key %s: %s." % (str(metadata['rating_key']), e))
+                imgur_info = {'imgur_title': title, 'imgur_url': imgur_url}
 
-    return poster_info
+    return imgur_info
+
+
+def set_hash_image_info(img=None, rating_key=None, width=600, height=1000,
+                        opacity=100, background='000000', blur=0, fallback=None):
+    if not rating_key and not img:
+        return fallback
+
+    if rating_key and not img:
+        if fallback == 'art':
+            img = '/library/metadata/{}/art'.format(rating_key)
+        else:
+            img = '/library/metadata/{}/thumb'.format(rating_key)
+
+    img_split = img.split('/')
+    img = '/'.join(img_split[:5])
+    rating_key = rating_key or img_split[3]
+
+    img_string = '{}.{}.{}.{}.{}.{}.{}.{}'.format(
+        plexpy.CONFIG.PMS_UUID, img, rating_key, width, height, opacity, background, blur, fallback)
+    img_hash = hashlib.sha256(img_string).hexdigest()
+
+    keys = {'img_hash': img_hash}
+    values = {'img': img,
+              'rating_key': rating_key,
+              'width': width,
+              'height': height,
+              'opacity': opacity,
+              'background': background,
+              'blur': blur,
+              'fallback': fallback}
+
+    db = database.MonitorDatabase()
+    db.upsert('image_hash_lookup', key_dict=keys, value_dict=values)
+
+    return img_hash
+
+
+def get_hash_image_info(img_hash=None):
+    db = database.MonitorDatabase()
+    query = 'SELECT * FROM image_hash_lookup WHERE img_hash = ?'
+    result = db.select_single(query, args=[img_hash])
+    return result
 
 
 def lookup_tvmaze_by_id(rating_key=None, thetvdb_id=None, imdb_id=None):
@@ -1287,13 +1340,13 @@ class CustomFormatter(Formatter):
 
     def format_field(self, value, format_spec):
         if format_spec.startswith('[') and format_spec.endswith(']'):
-            pattern = re.compile(r'\[(\d*):?(\d*)\]')
+            pattern = re.compile(r'\[(-?\d*):?(-?\d*)\]')
             if re.match(pattern, format_spec):  # slice
                 items = [x.strip() for x in unicode(value).split(',')]
                 slice_start, slice_end = re.search(pattern, format_spec).groups()
-                slice_start = max(int(slice_start), 0) if slice_start else None
-                slice_end = min(int(slice_end), len(items)) if slice_end else None
-                return ', '.join(items[slice_start:slice_end])
+                slice_start = helpers.cast_to_int(slice_start) or None
+                slice_end = helpers.cast_to_int(slice_end) or None
+                return ', '.join(items[slice(slice_start, slice_end)])
             else:
                 return value
         else:

plexpy/notifiers.py

@@ -338,7 +338,7 @@ def get_agent_class(agent_id=None, config=None):
         agent_id = int(agent_id)
 
         if agent_id == 0:
-            return GROWL(config=config,)
+            return GROWL(config=config)
         elif agent_id == 1:
             return PROWL(config=config)
         elif agent_id == 2:
@@ -431,9 +431,9 @@ def delete_notifier(notifier_id=None):
     db = database.MonitorDatabase()
 
     if str(notifier_id).isdigit():
-        logger.debug(u"Tautulli Notifiers :: Deleting notifier_id %s from the database." % notifier_id)
-        result = db.action('DELETE FROM notifiers WHERE id = ?',
-                                   args=[notifier_id])
+        logger.debug(u"Tautulli Notifiers :: Deleting notifier_id %s from the database."
+                     % notifier_id)
+        result = db.action('DELETE FROM notifiers WHERE id = ?', args=[notifier_id])
         return True
     else:
         return False
@@ -443,17 +443,18 @@ def get_notifier_config(notifier_id=None):
     if str(notifier_id).isdigit():
         notifier_id = int(notifier_id)
     else:
-        logger.error(u"Tautulli Notifiers :: Unable to retrieve notifier config: invalid notifier_id %s." % notifier_id)
+        logger.error(u"Tautulli Notifiers :: Unable to retrieve notifier config: invalid notifier_id %s."
+                     % notifier_id)
         return None
 
     db = database.MonitorDatabase()
-    result = db.select_single('SELECT * FROM notifiers WHERE id = ?',
-                                      args=[notifier_id])
+    result = db.select_single('SELECT * FROM notifiers WHERE id = ?', args=[notifier_id])
+
     if not result:
         return None
 
     try:
-        config = json.loads(result.pop('notifier_config') or '{}')
+        config = json.loads(result.pop('notifier_config', '{}'))
         notifier_agent = get_agent_class(agent_id=result['agent_id'], config=config)
         notifier_config = notifier_agent.return_config_options()
     except Exception as e:
@@ -490,15 +491,19 @@ def add_notifier_config(agent_id=None, **kwargs):
     if str(agent_id).isdigit():
         agent_id = int(agent_id)
     else:
-        logger.error(u"Tautulli Notifiers :: Unable to add new notifier: invalid agent_id %s." % agent_id)
+        logger.error(u"Tautulli Notifiers :: Unable to add new notifier: invalid agent_id %s."
+                     % agent_id)
         return False
 
     agent = next((a for a in available_notification_agents() if a['id'] == agent_id), None)
 
     if not agent:
-        logger.error(u"Tautulli Notifiers :: Unable to retrieve new notification agent: invalid agent_id %s." % agent_id)
+        logger.error(u"Tautulli Notifiers :: Unable to retrieve new notification agent: invalid agent_id %s."
+                     % agent_id)
         return False
 
+    agent_class = get_agent_class(agent_id=agent['id'])
+
     keys = {'id': None}
     values = {'agent_id': agent['id'],
               'agent_name': agent['name'],
@@ -508,6 +513,7 @@ def add_notifier_config(agent_id=None, **kwargs):
               'custom_conditions': json.dumps(DEFAULT_CUSTOM_CONDITIONS),
               'custom_conditions_logic': ''
               }
+
     if agent['name'] == 'scripts':
         for a in available_notification_actions():
             values[a['name'] + '_subject'] = ''
@@ -521,7 +527,8 @@ def add_notifier_config(agent_id=None, **kwargs):
     try:
         db.upsert(table_name='notifiers', key_dict=keys, value_dict=values)
         notifier_id = db.last_insert_id()
-        logger.info(u"Tautulli Notifiers :: Added new notification agent: %s (notifier_id %s)." % (agent['label'], notifier_id))
+        logger.info(u"Tautulli Notifiers :: Added new notification agent: %s (notifier_id %s)."
+                    % (agent['label'], notifier_id))
         blacklist_logger()
         return notifier_id
     except Exception as e:
@@ -533,13 +540,15 @@ def set_notifier_config(notifier_id=None, agent_id=None, **kwargs):
     if str(agent_id).isdigit():
         agent_id = int(agent_id)
     else:
-        logger.error(u"Tautulli Notifiers :: Unable to set exisiting notifier: invalid agent_id %s." % agent_id)
+        logger.error(u"Tautulli Notifiers :: Unable to set exisiting notifier: invalid agent_id %s."
+                     % agent_id)
         return False
 
     agent = next((a for a in available_notification_agents() if a['id'] == agent_id), None)
 
     if not agent:
-        logger.error(u"Tautulli Notifiers :: Unable to retrieve existing notification agent: invalid agent_id %s." % agent_id)
+        logger.error(u"Tautulli Notifiers :: Unable to retrieve existing notification agent: invalid agent_id %s."
+                     % agent_id)
         return False
 
     notify_actions = get_notify_actions()
@@ -553,7 +562,8 @@ def set_notifier_config(notifier_id=None, agent_id=None, **kwargs):
                  for k in kwargs.keys() if k.startswith(notify_actions) and k.endswith('_body')}
     notifier_config = {k[len(config_prefix):]: kwargs.pop(k)
                        for k in kwargs.keys() if k.startswith(config_prefix)}
-    notifier_config = get_agent_class(agent['id']).set_config(config=notifier_config)
+
+    agent_class = get_agent_class(agent_id=agent['id'], config=notifier_config)
 
     keys = {'id': notifier_id}
     values = {'agent_id': agent['id'],
@@ -571,7 +581,8 @@ def set_notifier_config(notifier_id=None, agent_id=None, **kwargs):
     db = database.MonitorDatabase()
     try:
         db.upsert(table_name='notifiers', key_dict=keys, value_dict=values)
-        logger.info(u"Tautulli Notifiers :: Updated notification agent: %s (notifier_id %s)." % (agent['label'], notifier_id))
+        logger.info(u"Tautulli Notifiers :: Updated notification agent: %s (notifier_id %s)."
+                    % (agent['label'], notifier_id))
         blacklist_logger()
 
         if agent['name'] == 'browser':
@@ -743,25 +754,33 @@ class Notifier(object):
     _DEFAULT_CONFIG = {}
 
     def __init__(self, config=None):
-        self.set_config(config)
+        self.config = self.set_config(config=config, default=self._DEFAULT_CONFIG)
 
-    def set_config(self, config=None):
-        self.config = self._validate_config(config)
-        return self.config
+    def set_config(self, config=None, default=None):
+        return self._validate_config(config=config, default=default)
 
-    def _validate_config(self, config=None):
+    def _validate_config(self, config=None, default=None):
         if config is None:
-            return self._DEFAULT_CONFIG
+            return default
 
         new_config = {}
-        for k, v in self._DEFAULT_CONFIG.iteritems():
+        for k, v in default.iteritems():
             if isinstance(v, int):
                 new_config[k] = helpers.cast_to_int(config.get(k, v))
+            elif isinstance(v, list):
+                c = config.get(k, v)
+                if not isinstance(c, list):
+                    new_config[k] = [c]
+                else:
+                    new_config[k] = c
             else:
                 new_config[k] = config.get(k, v)
 
         return new_config
 
+    def return_default_config(self):
+        return self._DEFAULT_CONFIG.copy()
+
     def notify(self, subject='', body='', action='', **kwargs):
         if self.NAME != 'Script':
             if not subject and self.config.get('incl_subject', True):
@@ -942,7 +961,7 @@ class ANDROIDAPP(Notifier):
                 'label': 'Device',
                 'description': 'No devices registered. '
                                '<a data-tab-destination="tabs-android_app" data-toggle="tab" data-dismiss="modal" '
-                               'style="cursor: pointer;">Get the Android App</a> and register a device.',
+                               'data-target="#top">Get the Android App</a> and register a device.',
                 'input_type': 'help'
                 })
         else:
@@ -952,7 +971,7 @@ class ANDROIDAPP(Notifier):
                 'name': 'androidapp_device_id',
                 'description': 'Set your Android app device or '
                                '<a data-tab-destination="tabs-android_app" data-toggle="tab" data-dismiss="modal" '
-                               'style="cursor: pointer;">register a new device</a> with Tautulli.',
+                               'data-target="#top">register a new device</a> with Tautulli.',
                 'input_type': 'select',
                 'select_options': devices
                 })
@@ -1205,7 +1224,9 @@ class DISCORD(Notifier):
                           'value': self.config['incl_card'],
                           'name': 'discord_incl_card',
                           'description': 'Include an info card with a poster and metadata with the notifications.<br>'
-                                         'Note: Imgur upload may need to be enabled under the notifications settings tab.',
+                                         'Note: <a data-tab-destination="tabs-notifications" data-dismiss="modal" '
+                                         'data-target="#notify_upload_posters">Image Hosting</a> '
+                                         'must be enabled under the notifications settings tab.',
                           'input_type': 'checkbox'
                           },
                          {'label': 'Include Plot Summaries',
@@ -1259,11 +1280,11 @@ class EMAIL(Notifier):
     Email notifications
     """
     NAME = 'Email'
-    _DEFAULT_CONFIG = {'from_name': '',
+    _DEFAULT_CONFIG = {'from_name': 'Tautulli',
                        'from': '',
-                       'to': '',
-                       'cc': '',
-                       'bcc': '',
+                       'to': [],
+                       'cc': [],
+                       'bcc': [],
                        'smtp_server': '',
                        'smtp_port': 25,
                        'smtp_user': '',
@@ -1272,16 +1293,6 @@ class EMAIL(Notifier):
                        'html_support': 1
                        }
 
-    def __init__(self, config=None):
-        super(EMAIL, self).__init__(config=config)
-
-        if not isinstance(self.config['to'], list):
-            self.config['to'] = [x.strip() for x in self.config['to'].split(';')]
-        if not isinstance(self.config['cc'], list):
-            self.config['cc'] = [x.strip() for x in self.config['cc'].split(';')]
-        if not isinstance(self.config['bcc'], list):
-            self.config['bcc'] = [x.strip() for x in self.config['bcc'].split(';')]
-
     def agent_notify(self, subject='', body='', action='', **kwargs):
         if self.config['html_support']:
             msg = MIMEMultipart('alternative')
@@ -1565,7 +1576,9 @@ class FACEBOOK(Notifier):
                           'value': self.config['incl_card'],
                           'name': 'facebook_incl_card',
                           'description': 'Include an info card with a poster and metadata with the notifications.<br>'
-                                         'Note: Imgur upload may need to be enabled under the notifications settings tab.',
+                                         'Note: <a data-tab-destination="tabs-notifications" data-dismiss="modal" '
+                                         'data-target="#notify_upload_posters">Image Hosting</a> '
+                                         'must be enabled under the notifications settings tab.',
                           'input_type': 'checkbox'
                           },
                          {'label': 'Movie Link Source',
@@ -1628,7 +1641,7 @@ class GROUPME(Notifier):
 
             if poster_content:
                 headers = {'X-Access-Token': self.config['access_token'],
-                           'Content-Type': 'image/jpeg'}
+                           'Content-Type': 'image/png'}
 
                 r = requests.post('https://image.groupme.com/pictures', headers=headers, data=poster_content)
 
@@ -1886,7 +1899,9 @@ class HIPCHAT(Notifier):
                           'value': self.config['incl_card'],
                           'name': 'hipchat_incl_card',
                           'description': 'Include an info card with a poster and metadata with the notifications.<br>'
-                                         'Note: Imgur upload may need to be enabled under the notifications settings tab.<br>'
+                                         'Note: <a data-tab-destination="tabs-notifications" data-dismiss="modal" '
+                                         'data-target="#notify_upload_posters">Image Hosting</a> '
+                                         'must be enabled under the notifications settings tab.<br>'
                                          'Note: This will change the notification type to HTML and emoticons will no longer work.',
                           'input_type': 'checkbox'
                           },
@@ -1992,7 +2007,7 @@ class JOIN(Notifier):
     """
     NAME = 'Join'
     _DEFAULT_CONFIG = {'api_key': '',
-                       'device_names': '',
+                       'device_names': [],
                        'priority': 2,
                        'incl_subject': 1,
                        'incl_poster': 0,
@@ -2001,12 +2016,6 @@ class JOIN(Notifier):
                        'music_provider': ''
                        }
 
-    def __init__(self, config=None):
-        super(JOIN, self).__init__(config=config)
-
-        if not isinstance(self.config['device_names'], list):
-            self.config['device_names'] = [x.strip() for x in self.config['device_names'].split(',')]
-
     def agent_notify(self, subject='', body='', action='', **kwargs):
         data = {'apikey': self.config['api_key'],
                 'deviceNames': ','.join(self.config['device_names']),
@@ -2111,7 +2120,9 @@ class JOIN(Notifier):
                           'value': self.config['incl_poster'],
                           'name': 'join_incl_poster',
                           'description': 'Include a poster with the notifications.<br>'
-                                         'Note: Imgur upload may need to be enabled under the notifications settings tab.',
+                                         'Note: <a data-tab-destination="tabs-notifications" data-dismiss="modal" '
+                                         'data-target="#notify_upload_posters">Image Hosting</a> '
+                                         'must be enabled under the notifications settings tab.',
                           'input_type': 'checkbox'
                           },
                          {'label': 'Movie Link Source',
@@ -2309,7 +2320,7 @@ class OSX(Notifier):
                        }
 
     def __init__(self, config=None):
-        self.set_config(config)
+        super(OSX, self).__init__(config=config)
 
         try:
             self.objc = __import__("objc")
@@ -2621,9 +2632,9 @@ class PUSHBULLET(Notifier):
                 logger.error(u"Tautulli Notifiers :: Unable to retrieve image for {name}.".format(name=self.NAME))
 
             if poster_content:
-                poster_filename = 'poster_{}.jpg'.format(pretty_metadata.parameters['rating_key'])
-                file_json = {'file_name': poster_filename, 'file_type': 'image/jpeg'}
-                files = {'file': (poster_filename, poster_content, 'image/jpeg')}
+                poster_filename = 'poster_{}.png'.format(pretty_metadata.parameters['rating_key'])
+                file_json = {'file_name': poster_filename, 'file_type': 'image/png'}
+                files = {'file': (poster_filename, poster_content, 'image/png')}
 
                 r = requests.post('https://api.pushbullet.com/v2/upload-request', headers=headers, json=file_json)
 
@@ -2777,8 +2788,8 @@ class PUSHOVER(Notifier):
                 logger.error(u"Tautulli Notifiers :: Unable to retrieve image for {name}.".format(name=self.NAME))
 
             if poster_content:
-                poster_filename = 'poster_{}.jpg'.format(pretty_metadata.parameters['rating_key'])
-                files = {'attachment': (poster_filename, poster_content, 'image/jpeg')}
+                poster_filename = 'poster_{}.png'.format(pretty_metadata.parameters['rating_key'])
+                files = {'attachment': (poster_filename, poster_content, 'image/png')}
                 headers = {}
 
         return self.make_request('https://api.pushover.net/1/messages.json', headers=headers, data=data, files=files)
@@ -2908,7 +2919,8 @@ class SCRIPTS(Notifier):
                        }
 
     def __init__(self, config=None):
-        self.set_config(config)
+        super(SCRIPTS, self).__init__(config=config)
+
         self.script_exts = {'.bat': '',
                             '.cmd': '',
                             '.exe': '',
@@ -3219,7 +3231,9 @@ class SLACK(Notifier):
                           'value': self.config['incl_card'],
                           'name': 'slack_incl_card',
                           'description': 'Include an info card with a poster and metadata with the notifications.<br>'
-                                         'Note: Imgur upload may need to be enabled under the notifications settings tab.',
+                                         'Note: <a data-tab-destination="tabs-notifications" data-dismiss="modal" '
+                                         'data-target="#notify_upload_posters">Image Hosting</a> '
+                                         'must be enabled under the notifications settings tab.',
                           'input_type': 'checkbox'
                           },
                          {'label': 'Include Plot Summaries',
@@ -3305,8 +3319,8 @@ class TELEGRAM(Notifier):
                 logger.error(u"Tautulli Notifiers :: Unable to retrieve image for {name}.".format(name=self.NAME))
 
             if poster_content:
-                poster_filename = 'poster_{}.jpg'.format(pretty_metadata.parameters['rating_key'])
-                files = {'photo': (poster_filename, poster_content, 'image/jpeg')}
+                poster_filename = 'poster_{}.png'.format(pretty_metadata.parameters['rating_key'])
+                files = {'photo': (poster_filename, poster_content, 'image/png')}
 
                 if len(text) > 200:
                     data['disable_notification'] = True
@@ -3458,7 +3472,9 @@ class TWITTER(Notifier):
                           'value': self.config['incl_poster'],
                           'name': 'twitter_incl_poster',
                           'description': 'Include a poster with the notifications.<br>'
-                                         'Note: Imgur upload may need to be enabled under the notifications settings tab.',
+                                         'Note: <a data-tab-destination="tabs-notifications" data-dismiss="modal" '
+                                         'data-target="#notify_upload_posters">Image Hosting</a> '
+                                         'must be enabled under the notifications settings tab.',
                           'input_type': 'checkbox'
                           }
                          ]

plexpy/plextv.py

@@ -266,6 +266,14 @@ class PlexTV(object):
 
         return request
 
+    def get_plextv_shared_servers(self, machine_id='', output_format=''):
+        uri = '/api/servers/%s/shared_servers' % machine_id
+        request = self.request_handler.make_request(uri=uri,
+                                                    request_type='GET',
+                                                    output_format=output_format)
+
+        return request
+
     def get_plextv_sync_lists(self, machine_id='', output_format=''):
         uri = '/servers/%s/sync_lists' % machine_id
         request = self.request_handler.make_request(uri=uri,
@@ -329,15 +337,18 @@ class PlexTV(object):
         return request
 
     def get_full_users_list(self):
-        friends_list = self.get_plextv_friends(output_format='xml')
         own_account = self.get_plextv_user_details(output_format='xml')
+        friends_list = self.get_plextv_friends(output_format='xml')
+        shared_servers = self.get_plextv_shared_servers(machine_id=plexpy.CONFIG.PMS_IDENTIFIER,
+                                                        output_format='xml')
+
         users_list = []
 
         try:
             xml_head = own_account.getElementsByTagName('user')
         except Exception as e:
             logger.warn(u"Tautulli PlexTV :: Unable to parse own account XML for get_full_users_list: %s." % e)
-            return {}
+            return []
 
         for a in xml_head:
             own_details = {"user_id": helpers.get_xml_attr(a, 'id'),
@@ -346,13 +357,16 @@ class PlexTV(object):
                            "email": helpers.get_xml_attr(a, 'email'),
                            "is_home_user": helpers.get_xml_attr(a, 'home'),
                            "is_admin": 1,
-                           "is_allow_sync": None,
+                           "is_allow_sync": 1,
                            "is_restricted": helpers.get_xml_attr(a, 'restricted'),
                            "filter_all": helpers.get_xml_attr(a, 'filterAll'),
                            "filter_movies": helpers.get_xml_attr(a, 'filterMovies'),
                            "filter_tv": helpers.get_xml_attr(a, 'filterTelevision'),
                            "filter_music": helpers.get_xml_attr(a, 'filterMusic'),
-                           "filter_photos": helpers.get_xml_attr(a, 'filterPhotos')
+                           "filter_photos": helpers.get_xml_attr(a, 'filterPhotos'),
+                           "user_token": helpers.get_xml_attr(a, 'authToken'),
+                           "server_token": helpers.get_xml_attr(a, 'authToken'),
+                           "shared_libraries": None,
                            }
 
             users_list.append(own_details)
@@ -361,7 +375,7 @@ class PlexTV(object):
             xml_head = friends_list.getElementsByTagName('User')
         except Exception as e:
             logger.warn(u"Tautulli PlexTV :: Unable to parse friends list XML for get_full_users_list: %s." % e)
-            return {}
+            return []
 
         for a in xml_head:
             friend = {"user_id": helpers.get_xml_attr(a, 'id'),
@@ -381,6 +395,28 @@ class PlexTV(object):
 
             users_list.append(friend)
 
+        try:
+            xml_head = shared_servers.getElementsByTagName('SharedServer')
+        except Exception as e:
+            logger.warn(u"Tautulli PlexTV :: Unable to parse shared server list XML for get_full_users_list: %s." % e)
+            return []
+
+        user_map = {}
+        for a in xml_head:
+            user_id = helpers.get_xml_attr(a, 'userID')
+            server_token = helpers.get_xml_attr(a, 'accessToken')
+
+            sections = a.getElementsByTagName('Section')
+            shared_libraries = [helpers.get_xml_attr(s, 'key')
+                                for s in sections if helpers.get_xml_attr(s, 'shared') == '1']
+
+            user_map[user_id] = {'server_token': server_token,
+                                 'shared_libraries': shared_libraries}
+
+        for u in users_list:
+            d = user_map.get(u['user_id'], {})
+            u.update(d)
+
         return users_list
 
     def get_synced_items(self, machine_id=None, client_id_filter=None, user_id_filter=None,

plexpy/pmsconnect.py

@@ -139,6 +139,22 @@ class PmsConnect(object):
 
         return request
 
+    def get_metadata_grandchildren(self, rating_key='', output_format=''):
+        """
+        Return metadata for graandchildren of the request item.
+
+        Parameters required:    rating_key { Plex ratingKey }
+        Optional parameters:    output_format { dict, json }
+
+        Output: array
+        """
+        uri = '/library/metadata/' + rating_key + '/grandchildren'
+        request = self.request_handler.make_request(uri=uri,
+                                                    request_type='GET',
+                                                    output_format=output_format)
+
+        return request
+
     def get_recently_added(self, start='0', count='0', output_format=''):
         """
         Return list of recently added items.
@@ -171,22 +187,6 @@ class PmsConnect(object):
 
         return request
 
-    def get_children_list(self, rating_key='', output_format=''):
-        """
-        Return list of children in requested library item.
-
-        Parameters required:    rating_key { ratingKey of parent }
-        Optional parameters:    output_format { dict, json }
-
-        Output: array
-        """
-        uri = '/library/metadata/' + rating_key + '/children'
-        request = self.request_handler.make_request(uri=uri,
-                                                    request_type='GET',
-                                                    output_format=output_format)
-
-        return request
-
     def get_children_list_related(self, rating_key='', output_format=''):
         """
         Return list of related children in requested collection item.
@@ -470,59 +470,86 @@ class PmsConnect(object):
                     output = {'recently_added': []}
                     return output
 
+            recents_main = []
             if a.getElementsByTagName('Directory'):
-                recents_main = a.getElementsByTagName('Directory')
-                for item in recents_main:
-                    recent_items = {'media_type': helpers.get_xml_attr(item, 'type'),
-                                    'rating_key': helpers.get_xml_attr(item, 'ratingKey'),
-                                    'parent_rating_key': helpers.get_xml_attr(item, 'parentRatingKey'),
-                                    'grandparent_rating_key': helpers.get_xml_attr(item, 'grandparentRatingKey'),
-                                    'title': helpers.get_xml_attr(item, 'title'),
-                                    'parent_title': helpers.get_xml_attr(item, 'parentTitle'),
-                                    'grandparent_title': helpers.get_xml_attr(item, 'grandparentTitle'),
-                                    'sort_title': helpers.get_xml_attr(item, 'titleSort'),
-                                    'media_index': helpers.get_xml_attr(item, 'index'),
-                                    'parent_media_index': helpers.get_xml_attr(item, 'parentIndex'),
-                                    'section_id': section_id if section_id else helpers.get_xml_attr(item, 'librarySectionID'),
-                                    'library_name': helpers.get_xml_attr(item, 'librarySectionTitle'),
-                                    'year': helpers.get_xml_attr(item, 'year'),
-                                    'thumb': helpers.get_xml_attr(item, 'thumb'),
-                                    'parent_thumb': helpers.get_xml_attr(item, 'parentThumb'),
-                                    'grandparent_thumb': helpers.get_xml_attr(item, 'grandparentThumb'),
-                                    'added_at': helpers.get_xml_attr(item, 'addedAt'),
-                                    'child_count': helpers.get_xml_attr(item, 'childCount')
-                                    }
-                    recents_list.append(recent_items)
-
+                recents_main += a.getElementsByTagName('Directory')
             if a.getElementsByTagName('Video'):
-                recents_main = a.getElementsByTagName('Video')
-                for item in recents_main:
-                    recent_items = {'media_type': helpers.get_xml_attr(item, 'type'),
-                                    'rating_key': helpers.get_xml_attr(item, 'ratingKey'),
-                                    'parent_rating_key': helpers.get_xml_attr(item, 'parentRatingKey'),
-                                    'grandparent_rating_key': helpers.get_xml_attr(item, 'grandparentRatingKey'),
-                                    'title': helpers.get_xml_attr(item, 'title'),
-                                    'parent_title': helpers.get_xml_attr(item, 'parentTitle'),
-                                    'grandparent_title': helpers.get_xml_attr(item, 'grandparentTitle'),
-                                    'sort_title': helpers.get_xml_attr(item, 'titleSort'),
-                                    'media_index': helpers.get_xml_attr(item, 'index'),
-                                    'parent_media_index': helpers.get_xml_attr(item, 'parentIndex'),
-                                    'section_id': section_id if section_id else helpers.get_xml_attr(item, 'librarySectionID'),
-                                    'library_name': helpers.get_xml_attr(item, 'librarySectionTitle'),
-                                    'year': helpers.get_xml_attr(item, 'year'),
-                                    'thumb': helpers.get_xml_attr(item, 'thumb'),
-                                    'parent_thumb': helpers.get_xml_attr(item, 'parentThumb'),
-                                    'grandparent_thumb': helpers.get_xml_attr(item, 'grandparentThumb'),
-                                    'added_at': helpers.get_xml_attr(item, 'addedAt'),
-                                    'child_count': helpers.get_xml_attr(item, 'childCount')
+                recents_main += a.getElementsByTagName('Video')
+
+            for m in recents_main:
+                directors = []
+                writers = []
+                actors = []
+                genres = []
+                labels = []
+
+                if m.getElementsByTagName('Director'):
+                    for director in m.getElementsByTagName('Director'):
+                        directors.append(helpers.get_xml_attr(director, 'tag'))
+
+                if m.getElementsByTagName('Writer'):
+                    for writer in m.getElementsByTagName('Writer'):
+                        writers.append(helpers.get_xml_attr(writer, 'tag'))
+
+                if m.getElementsByTagName('Role'):
+                    for actor in m.getElementsByTagName('Role'):
+                        actors.append(helpers.get_xml_attr(actor, 'tag'))
+
+                if m.getElementsByTagName('Genre'):
+                    for genre in m.getElementsByTagName('Genre'):
+                        genres.append(helpers.get_xml_attr(genre, 'tag'))
+
+                if m.getElementsByTagName('Label'):
+                    for label in m.getElementsByTagName('Label'):
+                        labels.append(helpers.get_xml_attr(label, 'tag'))
+
+                recent_item = {'media_type': helpers.get_xml_attr(m, 'type'),
+                               'section_id': helpers.get_xml_attr(m, 'librarySectionID'),
+                               'library_name': helpers.get_xml_attr(m, 'librarySectionTitle'),
+                               'rating_key': helpers.get_xml_attr(m, 'ratingKey'),
+                               'parent_rating_key': helpers.get_xml_attr(m, 'parentRatingKey'),
+                               'grandparent_rating_key': helpers.get_xml_attr(m, 'grandparentRatingKey'),
+                               'title': helpers.get_xml_attr(m, 'title'),
+                               'parent_title': helpers.get_xml_attr(m, 'parentTitle'),
+                               'grandparent_title': helpers.get_xml_attr(m, 'grandparentTitle'),
+                               'sort_title': helpers.get_xml_attr(m, 'titleSort'),
+                               'media_index': helpers.get_xml_attr(m, 'index'),
+                               'parent_media_index': helpers.get_xml_attr(m, 'parentIndex'),
+                               'studio': helpers.get_xml_attr(m, 'studio'),
+                               'content_rating': helpers.get_xml_attr(m, 'contentRating'),
+                               'summary': helpers.get_xml_attr(m, 'summary'),
+                               'tagline': helpers.get_xml_attr(m, 'tagline'),
+                               'rating': helpers.get_xml_attr(m, 'rating'),
+                               'audience_rating': helpers.get_xml_attr(m, 'audienceRating'),
+                               'user_rating': helpers.get_xml_attr(m, 'userRating'),
+                               'duration': helpers.get_xml_attr(m, 'duration'),
+                               'year': helpers.get_xml_attr(m, 'year'),
+                               'thumb': helpers.get_xml_attr(m, 'thumb'),
+                               'parent_thumb': helpers.get_xml_attr(m, 'parentThumb'),
+                               'grandparent_thumb': helpers.get_xml_attr(m, 'grandparentThumb'),
+                               'art': helpers.get_xml_attr(m, 'art'),
+                               'banner': helpers.get_xml_attr(m, 'banner'),
+                               'originally_available_at': helpers.get_xml_attr(m, 'originallyAvailableAt'),
+                               'added_at': helpers.get_xml_attr(m, 'addedAt'),
+                               'updated_at': helpers.get_xml_attr(m, 'updatedAt'),
+                               'last_viewed_at': helpers.get_xml_attr(m, 'lastViewedAt'),
+                               'guid': helpers.get_xml_attr(m, 'guid'),
+                               'directors': directors,
+                               'writers': writers,
+                               'actors': actors,
+                               'genres': genres,
+                               'labels': labels,
+                               'full_title': helpers.get_xml_attr(m, 'title'),
+                               'child_count': helpers.get_xml_attr(m, 'childCount')
                                }
-                    recents_list.append(recent_items)
+
+                recents_list.append(recent_item)
 
         output = {'recently_added': sorted(recents_list, key=lambda k: k['added_at'], reverse=True)}
 
         return output
 
-    def get_metadata_details(self, rating_key='', sync_id='', cache_key=None):
+    def get_metadata_details(self, rating_key='', sync_id='', cache_key=None, media_info=True):
         """
         Return processed and validated metadata list for requested item.
 
@@ -662,7 +689,8 @@ class PmsConnect(object):
                         'genres': genres,
                         'labels': labels,
                         'collections': collections,
-                        'full_title': helpers.get_xml_attr(metadata_main, 'title')
+                        'full_title': helpers.get_xml_attr(metadata_main, 'title'),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'show':
@@ -708,7 +736,8 @@ class PmsConnect(object):
                         'genres': genres,
                         'labels': labels,
                         'collections': collections,
-                        'full_title': helpers.get_xml_attr(metadata_main, 'title')
+                        'full_title': helpers.get_xml_attr(metadata_main, 'title'),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'season':
@@ -752,7 +781,8 @@ class PmsConnect(object):
                         'labels': show_details['labels'],
                         'collections': show_details['collections'],
                         'full_title': u'{} - {}'.format(helpers.get_xml_attr(metadata_main, 'parentTitle'),
-                                                        helpers.get_xml_attr(metadata_main, 'title'))
+                                                        helpers.get_xml_attr(metadata_main, 'title')),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'episode':
@@ -796,7 +826,8 @@ class PmsConnect(object):
                         'labels': show_details['labels'],
                         'collections': show_details['collections'],
                         'full_title': u'{} - {}'.format(helpers.get_xml_attr(metadata_main, 'grandparentTitle'),
-                                                        helpers.get_xml_attr(metadata_main, 'title'))
+                                                        helpers.get_xml_attr(metadata_main, 'title')),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'artist':
@@ -837,7 +868,8 @@ class PmsConnect(object):
                         'genres': genres,
                         'labels': labels,
                         'collections': collections,
-                        'full_title': helpers.get_xml_attr(metadata_main, 'title')
+                        'full_title': helpers.get_xml_attr(metadata_main, 'title'),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'album':
@@ -857,7 +889,7 @@ class PmsConnect(object):
                         'parent_media_index': helpers.get_xml_attr(metadata_main, 'parentIndex'),
                         'studio': helpers.get_xml_attr(metadata_main, 'studio'),
                         'content_rating': helpers.get_xml_attr(metadata_main, 'contentRating'),
-                        'summary': artist_details['summary'],
+                        'summary': helpers.get_xml_attr(metadata_main, 'summary') or artist_details['summary'],
                         'tagline': helpers.get_xml_attr(metadata_main, 'tagline'),
                         'rating': helpers.get_xml_attr(metadata_main, 'rating'),
                         'audience_rating': helpers.get_xml_attr(metadata_main, 'audienceRating'),
@@ -881,7 +913,8 @@ class PmsConnect(object):
                         'labels': labels,
                         'collections': collections,
                         'full_title': u'{} - {}'.format(helpers.get_xml_attr(metadata_main, 'parentTitle'),
-                                                        helpers.get_xml_attr(metadata_main, 'title'))
+                                                        helpers.get_xml_attr(metadata_main, 'title')),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'track':
@@ -925,7 +958,8 @@ class PmsConnect(object):
                         'labels': album_details['labels'],
                         'collections': album_details['collections'],
                         'full_title': u'{} - {}'.format(helpers.get_xml_attr(metadata_main, 'grandparentTitle'),
-                                                        helpers.get_xml_attr(metadata_main, 'title'))
+                                                        helpers.get_xml_attr(metadata_main, 'title')),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'photo_album':
@@ -966,7 +1000,8 @@ class PmsConnect(object):
                         'genres': genres,
                         'labels': labels,
                         'collections': collections,
-                        'full_title': helpers.get_xml_attr(metadata_main, 'title')
+                        'full_title': helpers.get_xml_attr(metadata_main, 'title'),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'photo':
@@ -1010,7 +1045,8 @@ class PmsConnect(object):
                         'labels': photo_album_details['labels'],
                         'collections': photo_album_details['collections'],
                         'full_title': u'{} - {}'.format(helpers.get_xml_attr(metadata_main, 'parentTitle'),
-                                                        helpers.get_xml_attr(metadata_main, 'title'))
+                                                        helpers.get_xml_attr(metadata_main, 'title')),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'collection':
@@ -1055,7 +1091,8 @@ class PmsConnect(object):
                         'genres': genres,
                         'labels': labels,
                         'collections': collections,
-                        'full_title': helpers.get_xml_attr(metadata_main, 'title')
+                        'full_title': helpers.get_xml_attr(metadata_main, 'title'),
+                        'children_count': helpers.get_xml_attr(metadata_main, 'leafCount')
                         }
 
         elif metadata_type == 'clip':
@@ -1104,7 +1141,7 @@ class PmsConnect(object):
         else:
             return {}
 
-        if metadata:
+        if metadata and media_info:
             medias = []
             media_items = metadata_main.getElementsByTagName('Media')
             for media in media_items:
@@ -1878,18 +1915,21 @@ class PmsConnect(object):
         else:
             return False
 
-    def get_item_children(self, rating_key=''):
+    def get_item_children(self, rating_key='', get_grandchildren=False):
         """
         Return processed and validated children list.
 
         Output: array
         """
-        children_data = self.get_children_list(rating_key, output_format='xml')
+        if get_grandchildren:
+            children_data = self.get_metadata_grandchildren(rating_key, output_format='xml')
+        else:
+            children_data = self.get_metadata_children(rating_key, output_format='xml')
 
         try:
             xml_head = children_data.getElementsByTagName('MediaContainer')
         except Exception as e:
-            logger.warn(u"Tautulli Pmsconnect :: Unable to parse XML for get_children_list: %s." % e)
+            logger.warn(u"Tautulli Pmsconnect :: Unable to parse XML for get_item_children: %s." % e)
             return []
 
         children_list = []
@@ -1912,20 +1952,71 @@ class PmsConnect(object):
             if a.getElementsByTagName('Track'):
                 result_data = a.getElementsByTagName('Track')
 
-            section_id = helpers.get_xml_attr(a, 'librarySectionID')
-
             if result_data:
-                for result in result_data:
-                    children_output = {'section_id': section_id,
-                                       'rating_key': helpers.get_xml_attr(result, 'ratingKey'),
-                                       'parent_rating_key': helpers.get_xml_attr(result, 'parentRatingKey'),
-                                       'media_index': helpers.get_xml_attr(result, 'index'),
-                                       'title': helpers.get_xml_attr(result, 'title'),
-                                       'parent_title': helpers.get_xml_attr(result, 'parentTitle'),
-                                       'year': helpers.get_xml_attr(result, 'year'),
-                                       'thumb': helpers.get_xml_attr(result, 'thumb'),
-                                       'parent_thumb': helpers.get_xml_attr(a, 'thumb'),
-                                       'duration': helpers.get_xml_attr(result, 'duration')
+                for m in result_data:
+                    directors = []
+                    writers = []
+                    actors = []
+                    genres = []
+                    labels = []
+
+                    if m.getElementsByTagName('Director'):
+                        for director in m.getElementsByTagName('Director'):
+                            directors.append(helpers.get_xml_attr(director, 'tag'))
+
+                    if m.getElementsByTagName('Writer'):
+                        for writer in m.getElementsByTagName('Writer'):
+                            writers.append(helpers.get_xml_attr(writer, 'tag'))
+
+                    if m.getElementsByTagName('Role'):
+                        for actor in m.getElementsByTagName('Role'):
+                            actors.append(helpers.get_xml_attr(actor, 'tag'))
+
+                    if m.getElementsByTagName('Genre'):
+                        for genre in m.getElementsByTagName('Genre'):
+                            genres.append(helpers.get_xml_attr(genre, 'tag'))
+
+                    if m.getElementsByTagName('Label'):
+                        for label in m.getElementsByTagName('Label'):
+                            labels.append(helpers.get_xml_attr(label, 'tag'))
+
+                    children_output = {'media_type': helpers.get_xml_attr(m, 'type'),
+                                       'section_id': helpers.get_xml_attr(m, 'librarySectionID'),
+                                       'library_name': helpers.get_xml_attr(m, 'librarySectionTitle'),
+                                       'rating_key': helpers.get_xml_attr(m, 'ratingKey'),
+                                       'parent_rating_key': helpers.get_xml_attr(m, 'parentRatingKey'),
+                                       'grandparent_rating_key': helpers.get_xml_attr(m, 'grandparentRatingKey'),
+                                       'title': helpers.get_xml_attr(m, 'title'),
+                                       'parent_title': helpers.get_xml_attr(m, 'parentTitle'),
+                                       'grandparent_title': helpers.get_xml_attr(m, 'grandparentTitle'),
+                                       'sort_title': helpers.get_xml_attr(m, 'titleSort'),
+                                       'media_index': helpers.get_xml_attr(m, 'index'),
+                                       'parent_media_index': helpers.get_xml_attr(m, 'parentIndex'),
+                                       'studio': helpers.get_xml_attr(m, 'studio'),
+                                       'content_rating': helpers.get_xml_attr(m, 'contentRating'),
+                                       'summary': helpers.get_xml_attr(m, 'summary'),
+                                       'tagline': helpers.get_xml_attr(m, 'tagline'),
+                                       'rating': helpers.get_xml_attr(m, 'rating'),
+                                       'audience_rating': helpers.get_xml_attr(m, 'audienceRating'),
+                                       'user_rating': helpers.get_xml_attr(m, 'userRating'),
+                                       'duration': helpers.get_xml_attr(m, 'duration'),
+                                       'year': helpers.get_xml_attr(m, 'year'),
+                                       'thumb': helpers.get_xml_attr(m, 'thumb'),
+                                       'parent_thumb': helpers.get_xml_attr(m, 'parentThumb'),
+                                       'grandparent_thumb': helpers.get_xml_attr(m, 'grandparentThumb'),
+                                       'art': helpers.get_xml_attr(m, 'art'),
+                                       'banner': helpers.get_xml_attr(m, 'banner'),
+                                       'originally_available_at': helpers.get_xml_attr(m, 'originallyAvailableAt'),
+                                       'added_at': helpers.get_xml_attr(m, 'addedAt'),
+                                       'updated_at': helpers.get_xml_attr(m, 'updatedAt'),
+                                       'last_viewed_at': helpers.get_xml_attr(m, 'lastViewedAt'),
+                                       'guid': helpers.get_xml_attr(m, 'guid'),
+                                       'directors': directors,
+                                       'writers': writers,
+                                       'actors': actors,
+                                       'genres': genres,
+                                       'labels': labels,
+                                       'full_title': helpers.get_xml_attr(m, 'title')
                                        }
                     children_list.append(children_output)
 
@@ -2162,7 +2253,7 @@ class PmsConnect(object):
         if str(section_id).isdigit():
             library_data = self.get_library_list(str(section_id), list_type, count, sort_type, label_key, output_format='xml')
         elif str(rating_key).isdigit():
-            library_data = self.get_children_list(str(rating_key), output_format='xml')
+            library_data = self.get_metadata_children(str(rating_key), output_format='xml')
         else:
             logger.warn(u"Tautulli Pmsconnect :: get_library_children called by invalid section_id or rating_key provided.")
             return []
@@ -2339,7 +2430,8 @@ class PmsConnect(object):
 
         return labels_list
 
-    def get_image(self, img=None, width='1000', height='1500', clip=False):
+    def get_image(self, img=None, width=600, height=1000, opacity=None, background=None, blur=None,
+                  img_format='png', clip=False, **kwargs):
         """
         Return image data as array.
         Array contains the image content type and image binary
@@ -2347,18 +2439,31 @@ class PmsConnect(object):
         Parameters required:    img { Plex image location }
         Optional parameters:    width { the image width }
                                 height { the image height }
+                                opacity { the image opacity 0-100 }
+                                background { the image background HEX }
+                                blur { the image blur 0-100 }
         Output: array
         """
 
+        width = width or 600
+        height = height or 1000
+
         if img:
             if clip:
                 params = {'url': '%s&%s' % (img, urllib.urlencode({'X-Plex-Token': self.token}))}
             else:
                 params = {'url': 'http://127.0.0.1:32400%s?%s' % (img, urllib.urlencode({'X-Plex-Token': self.token}))}
 
-            if width.isdigit() and height.isdigit():
             params['width'] = width
             params['height'] = height
+            params['format'] = img_format
+
+            if opacity:
+                params['opacity'] = opacity
+            if background:
+                params['background'] = background
+            if blur:
+                params['blur'] = blur
 
             uri = '/photo/:/transcode?%s' % urllib.urlencode(params)
             result = self.request_handler.make_request(uri=uri,

plexpy/users.py

@@ -21,9 +21,9 @@ import common
 import database
 import datatables
 import helpers
+import libraries
 import logger
 import plextv
-import pmsconnect
 import session
 
 
@@ -31,52 +31,32 @@ def refresh_users():
     logger.info(u"Tautulli Users :: Requesting users list refresh...")
     result = plextv.PlexTV().get_full_users_list()
 
-    monitor_db = database.MonitorDatabase()
-    user_data = Users()
-
     if result:
+        monitor_db = database.MonitorDatabase()
+
         for item in result:
 
-            shared_libraries = ''
-            user_tokens = user_data.get_tokens(user_id=item['user_id'])
-            if user_tokens and user_tokens['server_token']:
-                pms_connect = pmsconnect.PmsConnect(token=user_tokens['server_token'])
-                library_details = pms_connect.get_server_children()
+            if item.get('shared_libraries'):
+                item['shared_libraries'] = ';'.join(item['shared_libraries'])
+            elif item.get('server_token'):
+                libs = libraries.Libraries().get_sections()
+                item['shared_libraries'] = ';'.join([str(l['section_id']) for l in libs])
 
-                if library_details:
-                    shared_libraries = ';'.join(d['section_id'] for d in library_details['libraries_list'])
-                else:
-                    shared_libraries = ''
-
-            control_value_dict = {"user_id": item['user_id']}
-            new_value_dict = {"username": item['username'],
-                              "thumb": item['thumb'],
-                              "email": item['email'],
-                              "is_admin": item['is_admin'],
-                              "is_home_user": item['is_home_user'],
-                              "is_allow_sync": item['is_allow_sync'],
-                              "is_restricted": item['is_restricted'],
-                              "shared_libraries": shared_libraries,
-                              "filter_all": item['filter_all'],
-                              "filter_movies": item['filter_movies'],
-                              "filter_tv": item['filter_tv'],
-                              "filter_music": item['filter_music'],
-                              "filter_photos": item['filter_photos']
-                              }
+            keys_dict = {"user_id": item.pop('user_id')}
 
             # Check if we've set a custom avatar if so don't overwrite it.
-            if item['user_id']:
+            if keys_dict['user_id']:
                 avatar_urls = monitor_db.select('SELECT thumb, custom_avatar_url '
                                                 'FROM users WHERE user_id = ?',
-                                                [item['user_id']])
+                                                [keys_dict['user_id']])
                 if avatar_urls:
                     if not avatar_urls[0]['custom_avatar_url'] or \
                             avatar_urls[0]['custom_avatar_url'] == avatar_urls[0]['thumb']:
-                        new_value_dict['custom_avatar_url'] = item['thumb']
+                        item['custom_avatar_url'] = item['thumb']
                 else:
-                    new_value_dict['custom_avatar_url'] = item['thumb']
+                    item['custom_avatar_url'] = item['thumb']
 
-            monitor_db.upsert('users', new_value_dict, control_value_dict)
+            monitor_db.upsert('users', item, keys_dict)
 
         logger.info(u"Tautulli Users :: Users list refreshed.")
         return True

plexpy/version.py

@@ -1,2 +1,2 @@
 PLEXPY_BRANCH = "beta"
-PLEXPY_RELEASE_VERSION = "v2.0.26-beta"
+PLEXPY_RELEASE_VERSION = "v2.1.0-beta"

plexpy/webauth.py

@@ -20,6 +20,7 @@
 
 from datetime import datetime, timedelta
 import re
+from urllib import quote, unquote
 
 import cherrypy
 from hashing_passwords import check_hash
@@ -151,7 +152,11 @@ def check_auth(*args, **kwargs):
                     raise cherrypy.HTTPRedirect(plexpy.HTTP_ROOT)
 
         else:
-            raise cherrypy.HTTPRedirect(plexpy.HTTP_ROOT + "auth/logout")
+            redirect_uri = cherrypy.request.wsgi_environ['REQUEST_URI']
+            if redirect_uri:
+                redirect_uri = '?redirect_uri=' + quote(redirect_uri)
+
+            raise cherrypy.HTTPRedirect(plexpy.HTTP_ROOT + "auth/logout" + redirect_uri)
 
 
 def requireAuth(*conditions):
@@ -238,22 +243,22 @@ class AuthController(object):
         """Called on logout"""
         logger.debug(u"Tautulli WebAuth :: %s user '%s' logged out of Tautulli." % (user_group.capitalize(), username))
     
-    def get_loginform(self):
+    def get_loginform(self, redirect_uri=''):
         from plexpy.webserve import serve_template
-        return serve_template(templatename="login.html", title="Login")
+        return serve_template(templatename="login.html", title="Login", redirect_uri=unquote(redirect_uri))
     
     @cherrypy.expose
-    def index(self):
+    def index(self, *args, **kwargs):
         raise cherrypy.HTTPRedirect(plexpy.HTTP_ROOT + "auth/login")
 
     @cherrypy.expose
-    def login(self):
+    def login(self, redirect_uri='', *args, **kwargs):
         self.check_auth_enabled()
 
-        return self.get_loginform()
+        return self.get_loginform(redirect_uri=redirect_uri)
 
     @cherrypy.expose
-    def logout(self):
+    def logout(self, redirect_uri='', *args, **kwargs):
         self.check_auth_enabled()
 
         payload = check_jwt_token()
@@ -266,11 +271,15 @@ class AuthController(object):
         cherrypy.response.cookie[jwt_cookie]['path'] = '/'
 
         cherrypy.request.login = None
-        raise cherrypy.HTTPRedirect(plexpy.HTTP_ROOT + "auth/login")
+
+        if redirect_uri:
+            redirect_uri = '?redirect_uri=' + redirect_uri
+
+        raise cherrypy.HTTPRedirect(plexpy.HTTP_ROOT + "auth/login" + redirect_uri)
 
     @cherrypy.expose
     @cherrypy.tools.json_out()
-    def signin(self, username=None, password=None, remember_me='0', admin_login='0'):
+    def signin(self, username=None, password=None, remember_me='0', admin_login='0', *args, **kwargs):
         if cherrypy.request.method != 'POST':
             cherrypy.response.status = 405
             return {'status': 'error', 'message': 'Sign in using POST.'}

plexpy/webserve.py

@@ -41,6 +41,8 @@ import http_handler
 import libraries
 import log_reader
 import logger
+import newsletter_handler
+import newsletters
 import mobile_app
 import notification_handler
 import notifiers
@@ -286,11 +288,7 @@ class WebInterface(object):
 
     @cherrypy.expose
     @requireAuth(member_of("admin"))
-    def set_home_stats_config(self, refresh_interval=None, time_range=None, stats_type=None, stats_count=None,
-                              recently_added_count=None, **kwargs):
-        if refresh_interval:
-            plexpy.CONFIG.__setattr__('HOME_REFRESH_INTERVAL', refresh_interval)
-            plexpy.CONFIG.write()
+    def set_home_stats_config(self, time_range=None, stats_type=None, stats_count=None, recently_added_count=None, **kwargs):
         if time_range:
             plexpy.CONFIG.__setattr__('HOME_STATS_LENGTH', time_range)
             plexpy.CONFIG.write()
@@ -449,9 +447,9 @@ class WebInterface(object):
 
             Returns:
                 json:
-                    [{"section_id": 1, "section_name": "Movies"},
-                     {"section_id": 7, "section_name": "Music"},
-                     {"section_id": 2, "section_name": "TV Shows"},
+                    [{"section_id": 1, "section_name": "Movies", "section_type": "movie"},
+                     {"section_id": 7, "section_name": "Music", "section_type": "artist"},
+                     {"section_id": 2, "section_name": "TV Shows", "section_type": "show"},
                      {...}
                      ]
             ```
@@ -2318,8 +2316,8 @@ class WebInterface(object):
                     # Add traceback message to previous msg.
                     tl = (len(filt) - 1)
                     n = len(l) - len(l.lstrip(' '))
-                    l = ' ' * (2 * n) + l[n:]
-                    filt[tl][2] += '<br>' + l
+                    ll = '&nbsp;' * (2 * n) + unicode(l[n:], 'utf-8')
+                    filt[tl][2] += '<br>' + ll
                     continue
 
         log_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR']
@@ -2399,8 +2397,8 @@ class WebInterface(object):
                 None
 
             Optional parameters:
-                order_column (str):             "timestamp", "agent_name", "notify_action",
-                                                "subject_text", "body_text", "script_args"
+                order_column (str):             "timestamp", "notifier_id", "agent_name", "notify_action",
+                                                "subject_text", "body_text",
                 order_dir (str):                "desc" or "asc"
                 start (int):                    Row to start from, 0
                 length (int):                   Number of items to return, 25
@@ -2413,15 +2411,14 @@ class WebInterface(object):
                      "recordsFiltered": 163,
                      "data":
                         [{"agent_id": 13,
-                          "agent_name": "Telegram",
-                          "body_text": "Game of Thrones - S06E01 - The Red Woman [Transcode].",
+                          "agent_name": "telegram",
+                          "body_text": "DanyKhaleesi69 started playing The Red Woman.",
                           "id": 1000,
-                          "notify_action": "play",
-                          "poster_url": "http://i.imgur.com/ZSqS8Ri.jpg",
+                          "notify_action": "on_play",
                           "rating_key": 153037,
-                          "script_args": "[]",
                           "session_key": 147,
                           "subject_text": "Tautulli (Winterfell-Server)",
+                          "success": 1,
                           "timestamp": 1462253821,
                           "user": "DanyKhaleesi69",
                           "user_id": 8008135
@@ -2437,17 +2434,80 @@ class WebInterface(object):
         if not kwargs.get('json_data'):
             # TODO: Find some one way to automatically get the columns
             dt_columns = [("timestamp", True, True),
+                          ("notifier_id", True, True),
+                          ("agent_name", True, True),
+                          ("notify_action", True, True),
+                          ("subject_text", True, True),
+                          ("body_text", True, True)]
+            kwargs['json_data'] = build_datatables_json(kwargs, dt_columns, "timestamp")
+
+        data_factory = datafactory.DataFactory()
+        notification_logs = data_factory.get_notification_log(kwargs=kwargs)
+
+        return notification_logs
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @requireAuth(member_of("admin"))
+    @addtoapi()
+    def get_newsletter_log(self, **kwargs):
+        """ Get the data on the Tautulli newsletter logs table.
+
+            ```
+            Required parameters:
+                None
+
+            Optional parameters:
+                order_column (str):             "timestamp", "newsletter_id", "agent_name", "notify_action",
+                                                "subject_text", "start_date", "end_date", "uuid"
+                order_dir (str):                "desc" or "asc"
+                start (int):                    Row to start from, 0
+                length (int):                   Number of items to return, 25
+                search (str):                   A string to search for, "Telegram"
+
+            Returns:
+                json:
+                    {"draw": 1,
+                     "recordsTotal": 1039,
+                     "recordsFiltered": 163,
+                     "data":
+                        [{"agent_id": 0,
+                          "agent_name": "recently_added",
+                          "end_date": "2018-03-18",
+                          "id": 7,
+                          "newsletter_id": 1,
+                          "notify_action": "on_cron",
+                          "start_date": "2018-03-05",
+                          "subject_text": "Recently Added to Plex (Winterfell-Server)! (2018-03-18)",
+                          "success": 1,
+                          "timestamp": 1462253821,
+                          "uuid": "7fe4g65i"
+                          },
+                         {...},
+                         {...}
+                         ]
+                     }
+            ```
+        """
+        # Check if datatables json_data was received.
+        # If not, then build the minimal amount of json data for a query
+        if not kwargs.get('json_data'):
+            # TODO: Find some one way to automatically get the columns
+            dt_columns = [("timestamp", True, True),
+                          ("newsletter_id", True, True),
                           ("agent_name", True, True),
                           ("notify_action", True, True),
                           ("subject_text", True, True),
                           ("body_text", True, True),
-                          ("script_args", True, True)]
+                          ("start_date", True, True),
+                          ("end_date", True, True),
+                          ("uuid", True, True)]
             kwargs['json_data'] = build_datatables_json(kwargs, dt_columns, "timestamp")
 
         data_factory = datafactory.DataFactory()
-        notifications = data_factory.get_notification_log(kwargs=kwargs)
+        newsletter_logs = data_factory.get_newsletter_log(kwargs=kwargs)
 
-        return notifications
+        return newsletter_logs
 
     @cherrypy.expose
     @cherrypy.tools.json_out()
@@ -2474,6 +2534,31 @@ class WebInterface(object):
 
         return {'result': res, 'message': msg}
 
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @requireAuth(member_of("admin"))
+    @addtoapi()
+    def delete_newsletter_log(self, **kwargs):
+        """ Delete the Tautulli newsletter logs.
+
+            ```
+            Required paramters:
+                None
+
+            Optional parameters:
+                None
+
+            Returns:
+                None
+            ```
+        """
+        data_factory = datafactory.DataFactory()
+        result = data_factory.delete_newsletter_log()
+        res = 'success' if result else 'error'
+        msg = 'Cleared newsletter logs.' if result else 'Failed to clear newsletter logs.'
+
+        return {'result': res, 'message': msg}
+
     @cherrypy.expose
     @cherrypy.tools.json_out()
     @requireAuth(member_of("admin"))
@@ -2596,6 +2681,7 @@ class WebInterface(object):
             "https_key": plexpy.CONFIG.HTTPS_KEY,
             "https_domain": plexpy.CONFIG.HTTPS_DOMAIN,
             "https_ip": plexpy.CONFIG.HTTPS_IP,
+            "http_base_url": plexpy.CONFIG.HTTP_BASE_URL,
             "anon_redirect": plexpy.CONFIG.ANON_REDIRECT,
             "api_enabled": checked(plexpy.CONFIG.API_ENABLED),
             "api_key": plexpy.CONFIG.API_KEY,
@@ -2637,7 +2723,7 @@ class WebInterface(object):
             "refresh_users_on_startup": checked(plexpy.CONFIG.REFRESH_USERS_ON_STARTUP),
             "logging_ignore_interval": plexpy.CONFIG.LOGGING_IGNORE_INTERVAL,
             "notify_consecutive": checked(plexpy.CONFIG.NOTIFY_CONSECUTIVE),
-            "notify_upload_posters": checked(plexpy.CONFIG.NOTIFY_UPLOAD_POSTERS),
+            "notify_upload_posters": plexpy.CONFIG.NOTIFY_UPLOAD_POSTERS,
             "notify_recently_added_upgrade": checked(plexpy.CONFIG.NOTIFY_RECENTLY_ADDED_UPGRADE),
             "notify_group_recently_added_grandparent": checked(plexpy.CONFIG.NOTIFY_GROUP_RECENTLY_ADDED_GRANDPARENT),
             "notify_group_recently_added_parent": checked(plexpy.CONFIG.NOTIFY_GROUP_RECENTLY_ADDED_PARENT),
@@ -2647,6 +2733,7 @@ class WebInterface(object):
             "home_sections": json.dumps(plexpy.CONFIG.HOME_SECTIONS),
             "home_stats_cards": json.dumps(plexpy.CONFIG.HOME_STATS_CARDS),
             "home_library_cards": json.dumps(plexpy.CONFIG.HOME_LIBRARY_CARDS),
+            "home_refresh_interval": plexpy.CONFIG.HOME_REFRESH_INTERVAL,
             "buffer_threshold": plexpy.CONFIG.BUFFER_THRESHOLD,
             "buffer_wait": plexpy.CONFIG.BUFFER_WAIT,
             "group_history_tables": checked(plexpy.CONFIG.GROUP_HISTORY_TABLES),
@@ -2663,7 +2750,9 @@ class WebInterface(object):
             "music_watched_percent": plexpy.CONFIG.MUSIC_WATCHED_PERCENT,
             "themoviedb_lookup": checked(plexpy.CONFIG.THEMOVIEDB_LOOKUP),
             "tvmaze_lookup": checked(plexpy.CONFIG.TVMAZE_LOOKUP),
-            "show_advanced_settings": plexpy.CONFIG.SHOW_ADVANCED_SETTINGS
+            "show_advanced_settings": plexpy.CONFIG.SHOW_ADVANCED_SETTINGS,
+            "newsletter_dir": plexpy.CONFIG.NEWSLETTER_DIR,
+            "newsletter_self_hosted": checked(plexpy.CONFIG.NEWSLETTER_SELF_HOSTED)
         }
 
         return serve_template(templatename="settings.html", title="Settings", config=config, kwargs=kwargs)
@@ -2679,12 +2768,13 @@ class WebInterface(object):
             "grouping_global_history", "grouping_user_history", "grouping_charts", "group_history_tables",
             "pms_url_manual", "week_start_monday",
             "refresh_libraries_on_startup", "refresh_users_on_startup",
-            "notify_consecutive", "notify_upload_posters", "notify_recently_added_upgrade",
+            "notify_consecutive", "notify_recently_added_upgrade",
             "notify_group_recently_added_grandparent", "notify_group_recently_added_parent",
             "monitor_pms_updates", "monitor_remote_access", "get_file_sizes", "log_blacklist", "http_hash_password",
             "allow_guest_access", "cache_images", "http_proxy", "http_basic_auth", "notify_concurrent_by_ip",
             "history_table_activity", "plexpy_auto_update",
-            "themoviedb_lookup", "tvmaze_lookup", "http_plex_admin"
+            "themoviedb_lookup", "tvmaze_lookup", "http_plex_admin",
+            "newsletter_self_hosted"
         ]
         for checked_config in checked_configs:
             if checked_config not in kwargs:
@@ -3138,6 +3228,7 @@ class WebInterface(object):
         return parameters
 
     @cherrypy.expose
+    @cherrypy.tools.json_out()
     @requireAuth(member_of("admin"))
     def send_notification(self, notifier_id=None, subject='Tautulli', body='Test notification', notify_action='', **kwargs):
         """ Send a notification using Tautulli.
@@ -3163,21 +3254,20 @@ class WebInterface(object):
             notifier = notifiers.get_notifier_config(notifier_id=notifier_id)
 
             if notifier:
-                logger.debug(u"Sending %s%s notification." % (test, notifier['agent_name']))
-                if notification_handler.notify(notifier_id=notifier_id,
+                logger.debug(u"Sending %s%s notification." % (test, notifier['agent_label']))
+                notification_handler.add_notifier_each(notifier_id=notifier_id,
                                                        notify_action=notify_action,
                                                        subject=subject,
                                                        body=body,
-                                               **kwargs):
-                    return "Notification sent."
-                else:
-                    return "Notification failed."
+                                                       manual_trigger=True,
+                                                       **kwargs)
+                return {'result': 'success', 'message': 'Notification queued.'}
             else:
                 logger.debug(u"Unable to send %snotification, invalid notifier_id %s." % (test, notifier_id))
-                return "Invalid notifier id %s." % notifier_id
+                return {'result': 'success', 'message': 'Invalid notifier id %s.' % notifier_id}
         else:
             logger.debug(u"Unable to send %snotification, no notifier_id received." % test)
-            return "No notifier id received."
+            return {'result': 'success', 'message': 'No notifier id received.'}
 
     @cherrypy.expose
     @cherrypy.tools.json_out()
@@ -3827,7 +3917,6 @@ class WebInterface(object):
             return {'result': 'error', 'message': 'Notification failed.'}
 
     @cherrypy.expose
-    @requireAuth()
     def pms_image_proxy(self, **kwargs):
         """ See real_pms_image_proxy docs string"""
 
@@ -3840,7 +3929,8 @@ class WebInterface(object):
         return self.real_pms_image_proxy(**kwargs)
 
     @addtoapi('pms_image_proxy')
-    def real_pms_image_proxy(self, img='', rating_key=None, width='0', height='0',
+    def real_pms_image_proxy(self, img='', rating_key=None, width=0, height=0,
+                             opacity=100, background='000000', blur=0, img_format='png',
                              fallback=None, refresh=False, clip=False, **kwargs):
         """ Gets an image from the PMS and saves it to the image cache directory.
 
@@ -3851,8 +3941,12 @@ class WebInterface(object):
                 rating_key (str):       54321
 
             Optional parameters:
-                width (str):            150
-                height (str):           255
+                width (str):            300
+                height (str):           450
+                opacity (str):          25
+                background (str):       282828
+                blur (str):             3
+                img_format (str):       png
                 fallback (str):         "poster", "cover", "art"
                 refresh (bool):         True or False whether to refresh the image cache
 
@@ -3868,9 +3962,10 @@ class WebInterface(object):
             img = '/library/metadata/%s/thumb/1337' % rating_key
 
         img_string = img.rsplit('/', 1)[0] if '/library/metadata' in img else img
-        img_string += '%s%s' % (width, height)
+        img_string = '{}{}{}{}{}{}'.format(img_string, width, height, opacity, background, blur)
+
         fp = hashlib.md5(img_string).hexdigest()
-        fp += '.jpg'  # we want to be able to preview the thumbs
+        fp += '.%s' % img_format  # we want to be able to preview the thumbs
         c_dir = os.path.join(plexpy.CONFIG.CACHE_DIR, 'images')
         ffp = os.path.join(c_dir, fp)
 
@@ -3883,13 +3978,20 @@ class WebInterface(object):
             if not plexpy.CONFIG.CACHE_IMAGES or refresh or 'indexes' in img:
                 raise NotFound
 
-            return serve_file(path=ffp, content_type='image/jpeg')
+            return serve_file(path=ffp, content_type='image/png')
 
         except NotFound:
             # the image does not exist, download it from pms
             try:
                 pms_connect = pmsconnect.PmsConnect()
-                result = pms_connect.get_image(img, width, height, clip=clip)
+                result = pms_connect.get_image(img=img,
+                                               width=width,
+                                               height=height,
+                                               opacity=opacity,
+                                               background=background,
+                                               blur=blur,
+                                               img_format=img_format,
+                                               clip=clip)
 
                 if result and result[0]:
                     cherrypy.response.headers['Content-type'] = result[1]
@@ -3915,6 +4017,29 @@ class WebInterface(object):
                     fp = os.path.join(plexpy.PROG_DIR, 'data', fbi)
                     return serve_file(path=fp, content_type='image/png')
 
+    @cherrypy.expose
+    def image(self, *args, **kwargs):
+        if args:
+            img_hash = args[0]
+
+            if img_hash in ('poster', 'cover', 'art'):
+                if img_hash == 'poster':
+                    fbi = common.DEFAULT_POSTER_THUMB
+                elif img_hash == 'cover':
+                    fbi = common.DEFAULT_COVER_THUMB
+                elif img_hash == 'art':
+                    fbi = common.DEFAULT_ART
+
+                fp = os.path.join(plexpy.PROG_DIR, 'data', fbi)
+                return serve_file(path=fp, content_type='image/png')
+
+            img_info = notification_handler.get_hash_image_info(img_hash=img_hash)
+
+            if img_info:
+                kwargs.update(img_info)
+                return self.real_pms_image_proxy(**kwargs)
+
+        return
 
     @cherrypy.expose
     @requireAuth(member_of("admin"))
@@ -4051,7 +4176,7 @@ class WebInterface(object):
         """
 
         data_factory = datafactory.DataFactory()
-        result = data_factory.delete_poster_url(rating_key=rating_key)
+        result = data_factory.delete_imgur_info(rating_key=rating_key)
 
         if result:
             return {'result': 'success', 'message': 'Deleted Imgur poster.'}
@@ -5312,3 +5437,258 @@ class WebInterface(object):
     @requireAuth()
     def get_plexpy_url(self, **kwargs):
         return helpers.get_plexpy_url()
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @requireAuth(member_of("admin"))
+    @addtoapi()
+    def get_newsletters(self, **kwargs):
+        """ Get a list of configured newsletters.
+
+            ```
+            Required parameters:
+                None
+
+            Optional parameters:
+                None
+
+            Returns:
+                json:
+                    [{"id": 1,
+                      "agent_id": 13,
+                      "agent_name": "recently_added",
+                      "agent_label": "Recently Added",
+                      "friendly_name": "",
+                      "cron": "0 0 * * 1",
+                      "active": 1
+                      }
+                     ]
+            ```
+        """
+        result = newsletters.get_newsletters()
+        return result
+
+    @cherrypy.expose
+    @requireAuth(member_of("admin"))
+    def get_newsletters_table(self, **kwargs):
+        result = newsletters.get_newsletters()
+        return serve_template(templatename="newsletters_table.html", newsletters_list=result)
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @requireAuth(member_of("admin"))
+    @addtoapi()
+    def delete_newsletter(self, newsletter_id=None, **kwargs):
+        """ Remove a newsletter from the database.
+
+            ```
+            Required parameters:
+                newsletter_id (int):        The newsletter to delete
+
+            Optional parameters:
+                None
+
+            Returns:
+                None
+            ```
+        """
+        result = newsletters.delete_newsletter(newsletter_id=newsletter_id)
+        if result:
+            return {'result': 'success', 'message': 'Newsletter deleted successfully.'}
+        else:
+            return {'result': 'error', 'message': 'Failed to delete newsletter.'}
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @requireAuth(member_of("admin"))
+    @addtoapi()
+    def get_newsletter_config(self, newsletter_id=None, **kwargs):
+        """ Get the configuration for an existing notification agent.
+
+            ```
+            Required parameters:
+                newsletter_id (int):        The newsletter config to retrieve
+
+            Optional parameters:
+                None
+
+            Returns:
+                json:
+                    {"id": 1,
+                     "agent_id": 13,
+                     "agent_name": "recently_added",
+                     "agent_label": "Recently Added",
+                     "friendly_name": "",
+                     "cron": "0 0 * * 1",
+                     "active": 1
+                     "config": {"last_days": 7,
+                                "incl_libraries": [1, 2]
+                                },
+                     "email_config": {...},
+                     "config_options": [{...}, ...],
+                     "email_config_options": [{...}, ...]
+                     }
+            ```
+        """
+        result = newsletters.get_newsletter_config(newsletter_id=newsletter_id)
+        return result
+
+    @cherrypy.expose
+    @requireAuth(member_of("admin"))
+    def get_newsletter_config_modal(self, newsletter_id=None, **kwargs):
+        result = newsletters.get_newsletter_config(newsletter_id=newsletter_id)
+        return serve_template(templatename="newsletter_config.html", newsletter=result)
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @requireAuth(member_of("admin"))
+    @addtoapi()
+    def add_newsletter_config(self, agent_id=None, **kwargs):
+        """ Add a new notification agent.
+
+            ```
+            Required parameters:
+                agent_id (int):           The newsletter type to add
+
+            Optional parameters:
+                None
+
+            Returns:
+                None
+            ```
+        """
+        result = newsletters.add_newsletter_config(agent_id=agent_id, **kwargs)
+
+        if result:
+            return {'result': 'success', 'message': 'Added newsletter.', 'newsletter_id': result}
+        else:
+            return {'result': 'error', 'message': 'Failed to add newsletter.'}
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @requireAuth(member_of("admin"))
+    @addtoapi()
+    def set_newsletter_config(self, newsletter_id=None, agent_id=None, **kwargs):
+        """ Configure an exisitng notificaiton agent.
+
+            ```
+            Required parameters:
+                newsletter_id (int):            The newsletter config to update
+                agent_id (int):       The newsletter type of the newsletter
+
+            Optional parameters:
+                Pass all the config options for the agent with the agent prefix:
+                    e.g. For Recently Added: recently_added_last_days
+                                             recently_added_incl_movies
+                                             recently_added_incl_shows
+                                             recently_added_incl_artists
+
+            Returns:
+                None
+            ```
+        """
+        result = newsletters.set_newsletter_config(newsletter_id=newsletter_id,
+                                                   agent_id=agent_id,
+                                                   **kwargs)
+
+        if result:
+            return {'result': 'success', 'message': 'Saved newsletter.'}
+        else:
+            return {'result': 'error', 'message': 'Failed to save newsletter.'}
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @requireAuth(member_of("admin"))
+    @addtoapi("notify_newsletter")
+    def send_newsletter(self, newsletter_id=None, subject='', body='', message='', notify_action='', **kwargs):
+        """ Send a newsletter using Tautulli.
+
+            ```
+            Required parameters:
+                newsletter_id (int):      The ID number of the newsletter
+
+            Optional parameters:
+                None
+
+            Returns:
+                None
+            ```
+        """
+        cherrypy.response.headers['Cache-Control'] = "max-age=0,no-cache,no-store"
+
+        test = 'test ' if notify_action == 'test' else ''
+
+        if newsletter_id:
+            newsletter = newsletters.get_newsletter_config(newsletter_id=newsletter_id)
+
+            if newsletter:
+                logger.debug(u"Sending %s%s newsletter." % (test, newsletter['agent_label']))
+                newsletter_handler.add_newsletter_each(newsletter_id=newsletter_id,
+                                                       notify_action=notify_action,
+                                                       subject=subject,
+                                                       body=body,
+                                                       message=message,
+                                                        **kwargs)
+                return {'result': 'success', 'message': 'Newsletter queued.'}
+            else:
+                logger.debug(u"Unable to send %snewsletter, invalid newsletter_id %s." % (test, newsletter_id))
+                return {'result': 'error', 'message': 'Invalid newsletter id %s.' % newsletter_id}
+        else:
+            logger.debug(u"Unable to send %snotification, no newsletter_id received." % test)
+            return {'result': 'error', 'message': 'No newsletter id received.'}
+
+    @cherrypy.expose
+    def newsletter(self, *args, **kwargs):
+        if args:
+            if len(args) >= 2 and args[0] == 'image':
+                if args[1] == 'images':
+                    resource_dir = os.path.join(str(plexpy.PROG_DIR), 'data/interfaces/default/')
+                    try:
+                        return serve_file(path=os.path.join(resource_dir, *args[1:]), content_type='image/png')
+                    except NotFound:
+                        return
+
+                return self.image(args[1], refresh=True)
+
+            newsletter_uuid = args[0]
+            newsletter = newsletter_handler.get_newsletter(newsletter_uuid=newsletter_uuid)
+            return newsletter
+
+    @cherrypy.expose
+    @requireAuth(member_of("admin"))
+    def newsletter_preview(self, **kwargs):
+        kwargs['preview'] = 'true'
+        return serve_template(templatename="newsletter_preview.html",
+                              title="Newsletter",
+                              kwargs=kwargs)
+
+    @cherrypy.expose
+    @requireAuth(member_of("admin"))
+    def real_newsletter(self, newsletter_id=None, start_date=None, end_date=None,
+                        preview=False, master=False, raw=False, **kwargs):
+        if newsletter_id and newsletter_id != 'None':
+            newsletter = newsletters.get_newsletter_config(newsletter_id=newsletter_id)
+
+            if newsletter:
+                newsletter_agent = newsletters.get_agent_class(agent_id=newsletter['agent_id'],
+                                                               config=newsletter['config'],
+                                                               start_date=start_date,
+                                                               end_date=end_date,
+                                                               subject=newsletter['subject'],
+                                                               body=newsletter['body'],
+                                                               message=newsletter['message'])
+                preview = (preview == 'true')
+                master = (master == 'true')
+                raw = (raw == 'true')
+
+                if raw:
+                    cherrypy.response.headers['Content-Type'] = 'application/json;charset=UTF-8'
+                    return json.dumps(newsletter_agent.raw_data(preview=preview))
+
+                return newsletter_agent.generate_newsletter(preview=preview, master=master)
+
+            logger.error(u"Failed to retrieve newsletter: Invalid newsletter_id %s" % newsletter_id)
+            return "Failed to retrieve newsletter: invalid newsletter_id parameter"
+
+        logger.error(u"Failed to retrieve newsletter: Missing newsletter_id parameter.")
+        return "Failed to retrieve newsletter: missing newsletter_id parameter"