From 21ad5257cf5e92891a00e8290d64cbbc1370610c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Thu, 30 Jul 2020 13:53:40 +0200 Subject: [PATCH] add weight settings for track parameters --- nonstop/forms.py | 31 ++++++++++ nonstop/management/commands/stamina.py | 62 +++++++++---------- .../0030_nonstopzonesettings_weights_text.py | 20 ++++++ nonstop/models.py | 36 +++++++++++ nonstop/templates/nonstop/range_widget.html | 2 + nonstop/templates/nonstop/zone_settings.html | 20 ++++++ nonstop/urls.py | 5 +- nonstop/utils.py | 62 ++++++++++++++++++- nonstop/views.py | 35 ++++++++++- nonstop/widgets.py | 7 ++- 10 files changed, 242 insertions(+), 38 deletions(-) create mode 100644 nonstop/migrations/0030_nonstopzonesettings_weights_text.py create mode 100644 nonstop/templates/nonstop/range_widget.html diff --git a/nonstop/forms.py b/nonstop/forms.py index 33475a6..7deaa84 100644 --- a/nonstop/forms.py +++ b/nonstop/forms.py @@ -62,3 +62,34 @@ class ZoneSettingsForm(forms.Form): label=_('Jingles'), widget=widgets.JinglesWidget, choices=get_jingle_choices) + + weight_lang_en = forms.IntegerField( + label=_('Weight adjustment for English tracks'), + min_value=-10, + max_value=10, + widget=widgets.RangeWidget) + weight_lang_fr = forms.IntegerField( + label=_('Weight adjustment for French tracks'), + min_value=-10, + max_value=10, + widget=widgets.RangeWidget) + weight_lang_nl = forms.IntegerField( + label=_('Weight adjustment for Dutch tracks'), + min_value=-10, + max_value=10, + widget=widgets.RangeWidget) + weight_lang_other = forms.IntegerField( + label=_('Weight adjustment for tracks in other languages'), + min_value=-10, + max_value=10, + widget=widgets.RangeWidget) + weight_instru = forms.IntegerField( + label=_('Weight adjustment for instrumental tracks'), + min_value=-10, + max_value=10, + widget=widgets.RangeWidget) + weight_cfwb = forms.IntegerField( + label=_('Weight adjustment for FWB tracks'), + min_value=-10, + max_value=10, + widget=widgets.RangeWidget) diff --git a/nonstop/management/commands/stamina.py b/nonstop/management/commands/stamina.py index 768e455..e9eeb85 100644 --- a/nonstop/management/commands/stamina.py +++ b/nonstop/management/commands/stamina.py @@ -13,8 +13,10 @@ from django.conf import settings from django.core.management.base import BaseCommand from emissions.models import Nonstop -from nonstop.models import Track, Jingle, SomaLogLine, ScheduledDiffusion, RecurringStreamOccurence, RecurringRandomDirectoryOccurence +from nonstop.models import (NonstopZoneSettings, Track, Jingle, SomaLogLine, + ScheduledDiffusion, RecurringStreamOccurence, RecurringRandomDirectoryOccurence) from nonstop.app_settings import app_settings +from nonstop.utils import Tracklist logger = logging.getLogger('stamina') @@ -86,14 +88,24 @@ class Command(BaseCommand): playlist = [] adjustment_counter = 0 try: - jingles = list(zone.nonstopzonesettings_set.first().jingles.all()) + zone_settings = zone.nonstopzonesettings_set.first() + jingles = list(zone_settings.jingles.all()) except AttributeError: + zone_settings = NonstopZoneSettings() jingles = [] + zone_ids = [zone.id] + extra_zones = app_settings.EXTRA_ZONES.get(zone.slug) + if extra_zones: + zone_ids.extend([x.id for x in Nonstop.objects.filter(slug__in=extra_zones)]) + recent_tracks_id = [x.track_id for x in SomaLogLine.objects.exclude(on_air=False).filter( track__isnull=False, play_timestamp__gt=datetime.datetime.now() - datetime.timedelta(days=app_settings.NO_REPEAT_DELAY))] + + tracklist = Tracklist(zone_settings, zone_ids, recent_tracks_id) + random_tracks_iterator = tracklist.get_random_tracks() t0 = datetime.datetime.now() allow_overflow = False while current_datetime < end_datetime: @@ -103,67 +115,49 @@ class Command(BaseCommand): if jingles and current_datetime - self.last_jingle_datetime > datetime.timedelta(minutes=20): # jingle time, every ~20 minutes - playlist.append(random.choice(jingles)) + tracklist.playlist.append(random.choice(jingles)) self.last_jingle_datetime = current_datetime - current_datetime = start_datetime + sum( - [x.duration for x in playlist], datetime.timedelta(seconds=0)) - - zone_ids = [zone.id] - extra_zones = app_settings.EXTRA_ZONES.get(zone.slug) - if extra_zones: - zone_ids.extend([x.id for x in Nonstop.objects.filter(slug__in=extra_zones)]) + current_datetime = start_datetime + tracklist.get_duration() remaining_time = (end_datetime - current_datetime) - track = Track.objects.filter( - nonstop_zones__in=zone_ids, - duration__isnull=False).exclude( - id__in=recent_tracks_id + [x.id for x in playlist if isinstance(x, Track)] - ).order_by('?').first() - if track is None: - # no track, reduce recent tracks exclusion - recent_tracks_id = recent_tracks_id[:len(recent_tracks_id)//2] - continue - playlist.append(track) - current_datetime = start_datetime + sum( - [x.duration for x in playlist], datetime.timedelta(seconds=0)) + + track = next(random_tracks_iterator) + tracklist.append(track) + current_datetime = start_datetime + tracklist.get_duration() if current_datetime > end_datetime and not allow_overflow: # last track overshot # 1st strategy: remove last track and try to get a track with # exact remaining time logger.debug('Overshoot %s, %s', adjustment_counter, current_datetime) - playlist = playlist[:-1] + tracklist.pop() track = Track.objects.filter( - nonstop_zones=zone, + nonstop_zones__in=zone_ids, duration__gte=remaining_time, duration__lt=remaining_time + datetime.timedelta(seconds=1) - ).exclude( - id__in=recent_tracks_id + [x.id for x in playlist if isinstance(x, Track)] - ).order_by('?').first() + ).exclude(id__in=tracklist.get_recent_track_ids()).order_by('?').first() if track: # found a track - playlist.append(track) + tracklist.append(track) else: # fallback strategy: didn't find track of expected duration, # reduce playlist further adjustment_counter += 1 - playlist = playlist[:-1] - if len(playlist) == 0 or adjustment_counter > 5: + if tracklist.pop() is None or adjustment_counter > 5: # a dedicated sound that ended a bit too early, # or too many failures to get an appropriate file, # allow whatever comes. allow_overflow = True logger.debug('Allowing overflows') - current_datetime = start_datetime + sum( - [x.duration for x in playlist], datetime.timedelta(seconds=0)) + current_datetime = start_datetime + tracklist.get_duration() logger.info('Computed playlist for "%s" (computation time: %ss)', zone, (datetime.datetime.now() - t0)) current_datetime = start_datetime - for track in playlist: + for track in tracklist.playlist: logger.debug('- track: %s %s %s', current_datetime, track.duration, track.title) current_datetime += track.duration logger.debug('- end: %s', current_datetime) - return playlist + return tracklist.playlist def is_nonstop_on_air(self): # check if nonstop system is currently on air diff --git a/nonstop/migrations/0030_nonstopzonesettings_weights_text.py b/nonstop/migrations/0030_nonstopzonesettings_weights_text.py new file mode 100644 index 0000000..4b1c324 --- /dev/null +++ b/nonstop/migrations/0030_nonstopzonesettings_weights_text.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2020-07-30 12:11 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nonstop', '0029_scheduleddiffusion_auto_delayed'), + ] + + operations = [ + migrations.AddField( + model_name='nonstopzonesettings', + name='weights_text', + field=models.TextField(null=True), + ), + ] diff --git a/nonstop/models.py b/nonstop/models.py index 53834ea..c941d2b 100644 --- a/nonstop/models.py +++ b/nonstop/models.py @@ -1,4 +1,6 @@ +import collections import datetime +import json import os import random @@ -137,6 +139,27 @@ class Track(models.Model): if os.path.exists(zone_path): os.unlink(zone_path) + def match_criteria(self, criteria): + return getattr(self, 'match_criteria_' + criteria)() + + def match_criteria_lang_en(self): + return self.language == 'en' + + def match_criteria_lang_fr(self): + return self.language == 'fr' + + def match_criteria_lang_nl(self): + return self.language == 'nl' + + def match_criteria_lang_other(self): + return self.language == 'other' + + def match_criteria_instru(self): + return self.instru + + def match_criteria_cfwb(self): + return self.cfwb + class NonstopFile(models.Model): filepath = models.CharField(_('Filepath'), max_length=255) @@ -291,9 +314,22 @@ class NonstopZoneSettings(models.Model): intro_jingle = models.ForeignKey(Jingle, blank=True, null=True, related_name='+') jingles = models.ManyToManyField(Jingle, blank=True) + weights_text = models.TextField(null=True) # will be read as json + def __str__(self): return str(self.nonstop) + @property + def weights(self): + weights = collections.defaultdict(int) + for k, v in json.loads(self.weights_text or "{}").items(): + weights[k] = v + return weights + + @weights.setter + def weights(self, d): + self.weights_text = json.dumps(d) + class RecurringStreamDiffusion(models.Model): schedule = models.ForeignKey('emissions.Schedule', on_delete=models.CASCADE) diff --git a/nonstop/templates/nonstop/range_widget.html b/nonstop/templates/nonstop/range_widget.html new file mode 100644 index 0000000..a143bc8 --- /dev/null +++ b/nonstop/templates/nonstop/range_widget.html @@ -0,0 +1,2 @@ + + diff --git a/nonstop/templates/nonstop/zone_settings.html b/nonstop/templates/nonstop/zone_settings.html index f733778..e55cbcb 100644 --- a/nonstop/templates/nonstop/zone_settings.html +++ b/nonstop/templates/nonstop/zone_settings.html @@ -21,4 +21,24 @@ + + {% endblock %} diff --git a/nonstop/urls.py b/nonstop/urls.py index f4fccd7..8aa90c6 100644 --- a/nonstop/urls.py +++ b/nonstop/urls.py @@ -8,7 +8,7 @@ from .views import (SomaDayArchiveView, SomaDayArchiveCsvView, RedirectTodayView QuickLinksView, SearchView, CleanupView, ArtistTracksMetadataView, SearchCsvView, AddSomaDiffusionView, DelSomaDiffusionView, DiffusionPropertiesView, AjaxProgram, ZonesView, ZoneSettings, - jingle_audio_view, track_sound, + jingle_audio_view, track_sound, ZoneTracklistPercents, MuninTracks) urlpatterns = [ @@ -50,6 +50,9 @@ urlpatterns = [ # ajax parts url(r'^ajax/program/(?P[\d-]*)$', AjaxProgram.as_view(), name='nonstop-ajax-program'), + url(r'^ajax/zones/(?P[\w-]+)/percents/$', + ZoneTracklistPercents.as_view(), + name='nonstop-ajax-zone-percents'), ] public_urlpatterns = [ diff --git a/nonstop/utils.py b/nonstop/utils.py index f1abc27..cbc4996 100644 --- a/nonstop/utils.py +++ b/nonstop/utils.py @@ -1,5 +1,6 @@ import datetime import os +import random import shutil import socket import time @@ -9,7 +10,7 @@ from django.utils.timezone import now import xml.etree.ElementTree as ET from emissions.models import Diffusion, Schedule -from .models import SomaLogLine, ScheduledDiffusion, Jingle, RecurringStreamOccurence +from .models import SomaLogLine, ScheduledDiffusion, Jingle, RecurringStreamOccurence, Track from .app_settings import app_settings @@ -163,3 +164,62 @@ def get_palinsesto_xml(): palinsesto_bytes = palinsesto_bytes[9:] palinsesto_xml = ET.fromstring(palinsesto_bytes) return palinsesto_xml + + +class Tracklist: + def __init__(self, zone_settings, zone_ids, recent_tracks_id=None, filter_kwargs={}, k=30): + self.zone_settings = zone_settings + self.zone_ids = zone_ids + self.playlist = [] + self.recent_tracks_id = recent_tracks_id or [] + self.filter_kwargs = filter_kwargs + self.k = k + + def append(self, track): + # track or jingle + self.playlist.append(track) + + def pop(self): + return self.playlist.pop() if self.playlist else None + + def get_recent_track_ids(self): + return self.recent_tracks_id + [x.id for x in self.playlist if isinstance(x, Track)] + + def get_duration(self): + return sum([x.duration for x in self.playlist], datetime.timedelta(seconds=0)) + + def get_random_tracks(self, k=30): + weights = self.zone_settings.weights + + while True: + # pick tracks from db + tracks = Track.objects.filter( + nonstop_zones__in=self.zone_ids, + duration__isnull=False, + **self.filter_kwargs).exclude( + id__in=self.get_recent_track_ids() + ).order_by('?')[:k*10] + if len(tracks) == 0: + self.recent_tracks_id = self.recent_tracks_id[:len(self.recent_tracks_id) // 2] + continue + + def compute_weight(track): + weight = 0 + for weight_key, weight_value in weights.items(): + if track.match_criteria(weight_key): + weight += weight_value + if weight < 0: + weight = 1 + (weight / 20) + else: + weight = 1 + (weight / 2) + return weight + + track_weights = [compute_weight(x) for x in tracks] + tracks = random.choices(tracks, weights=track_weights, k=k) + + seen = set() + for track in tracks: + if track in seen: + continue + yield track + seen.add(track) diff --git a/nonstop/views.py b/nonstop/views.py index 0db85e9..5681596 100644 --- a/nonstop/views.py +++ b/nonstop/views.py @@ -1,3 +1,4 @@ +import collections import copy import csv import datetime @@ -14,7 +15,7 @@ from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.core.urlresolvers import reverse, reverse_lazy from django.contrib import messages from django.db.models import Q, Sum -from django.http import HttpResponse, HttpResponseRedirect, FileResponse, Http404 +from django.http import HttpResponse, HttpResponseRedirect, FileResponse, JsonResponse, Http404 from django.utils.six import StringIO from django.utils.translation import ugettext_lazy as _ from django.views.generic.base import RedirectView, TemplateView @@ -612,6 +613,8 @@ class ZoneSettings(FormView): initial['end'] = zone.end.strftime('%H:%M') if zone.end else None initial['intro_jingle'] = zone_settings.intro_jingle_id initial['jingles'] = [x.id for x in zone_settings.jingles.all()] + for key, value in zone_settings.weights.items(): + initial['weight_%s' % key] = value return initial def form_valid(self, form): @@ -623,6 +626,8 @@ class ZoneSettings(FormView): zone.end = form.cleaned_data['end'] zone_settings.jingles.set(form.cleaned_data['jingles']) zone_settings.intro_jingle_id = form.cleaned_data['intro_jingle'] + weights = {key[7:]: value for key, value in form.cleaned_data.items() if key.startswith('weight_')} + zone_settings.weights = weights zone.save() zone_settings.save() return super().form_valid(form) @@ -650,3 +655,31 @@ class MuninTracks(StatisticsView): active_tracks_qs.filter(language='fr').count() / active_tracks_qs.exclude(language__isnull=True).exclude(language__in=('', 'na')).count()) return context + + +class ZoneTracklistPercents(DetailView): + model = Nonstop + + def get(self, request, *args, **kwargs): + zone = self.get_object() + zone_settings = zone.nonstopzonesettings_set.first() + weights = {key[7:]: int(value) for key, value in request.GET.items() if key.startswith('weight_')} + zone_settings.weights = weights + + tracklist = utils.Tracklist(zone_settings, zone_ids=[zone.id]) + random_tracks_iterator = tracklist.get_random_tracks(k=100) + + counts = collections.defaultdict(int) + + for i, track in enumerate(random_tracks_iterator): + if i == 1000: + break + for weight in weights: + if track.match_criteria(weight): + counts[weight] += 1 + + data = {} + for weight in weights: + data[weight] = counts[weight] / 1000 + + return JsonResponse(data) diff --git a/nonstop/widgets.py b/nonstop/widgets.py index 6686589..b8a674a 100644 --- a/nonstop/widgets.py +++ b/nonstop/widgets.py @@ -1,4 +1,4 @@ -from django.forms.widgets import TimeInput, SelectMultiple +from django.forms.widgets import TimeInput, SelectMultiple, NumberInput from django.utils.safestring import mark_safe from .models import Jingle @@ -39,3 +39,8 @@ class JinglesWidget(SelectMultiple): if data.get('%s-%s' % (name, choice_id)): choices.append(choice_id) return choices + + +class RangeWidget(NumberInput): + input_type = 'range' + template_name = 'nonstop/range_widget.html' -- 2.39.2