Commit 98e5fdeb authored by Eliot Berriot's avatar Eliot Berriot

Merge branch 'release/0.5.3'

parents 9dc69ac1 41404a59
Changelog
=========
0.6 (Unreleased)
----------------
.. towncrier
0.5.3 (2018-02-27)
------------------
Features:
- Added admin interface for radios, track files, favorites and import requests (#80)
- Added basic instance stats on /about (#82)
- Search now unaccent letters for queries like "The Dø" or "Björk" yielding more results (#81)
Bugfixes:
- Always use username in sidebar (#89)
- Click event outside of player icons (#83)
- Fixed broken import because of missing transaction
- Now always load next radio track on last queue track ended (#87)
- Now exclude tracks without file from radio candidates (#88)
- skip to next track properly on 40X errors (#86)
Other:
- Switched to towncrier for changelog management and compilation
0.5.2 (2018-02-26)
......
......@@ -37,6 +37,7 @@ DJANGO_APPS = (
'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.postgres',
# Useful template tags:
# 'django.contrib.humanize',
......
# -*- coding: utf-8 -*-
__version__ = '0.5.2'
__version__ = '0.5.3'
__version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])
# Generated by Django 2.0.2 on 2018-02-27 18:43
from django.db import migrations
from django.contrib.postgres.operations import UnaccentExtension
class Migration(migrations.Migration):
dependencies = []
operations = [
UnaccentExtension()
]
import os
import shutil
from django.db import transaction
def rename_file(instance, field_name, new_name, allow_missing_file=False):
field = getattr(instance, field_name)
......@@ -17,3 +19,9 @@ def rename_file(instance, field_name, new_name, allow_missing_file=False):
field.name = os.path.join(initial_path, new_name_with_extension)
instance.save()
return new_name_with_extension
def on_commit(f, *args, **kwargs):
return transaction.on_commit(
lambda: f(*args, **kwargs)
)
from django.contrib import admin
from . import models
@admin.register(models.TrackFavorite)
class TrackFavoriteAdmin(admin.ModelAdmin):
list_display = ['user', 'track', 'creation_date']
list_select_related = [
'user',
'track'
]
......@@ -6,3 +6,7 @@ from . import models
class ListeningAdmin(admin.ModelAdmin):
list_display = ['track', 'end_date', 'user', 'session_key']
search_fields = ['track__name', 'user__username']
list_select_related = [
'user',
'track'
]
from django.db.models import Sum
from funkwhale_api.favorites.models import TrackFavorite
from funkwhale_api.history.models import Listening
from funkwhale_api.music import models
from funkwhale_api.users.models import User
def get():
return {
'users': get_users(),
'tracks': get_tracks(),
'albums': get_albums(),
'artists': get_artists(),
'track_favorites': get_track_favorites(),
'listenings': get_listenings(),
'music_duration': get_music_duration(),
}
def get_users():
return User.objects.count()
def get_listenings():
return Listening.objects.count()
def get_track_favorites():
return TrackFavorite.objects.count()
def get_tracks():
return models.Track.objects.count()
def get_albums():
return models.Album.objects.count()
def get_artists():
return models.Artist.objects.count()
def get_music_duration():
seconds = models.TrackFile.objects.aggregate(
d=Sum('duration'),
)['d']
if seconds:
return seconds / 3600
return 0
from django.conf.urls import url
from django.views.decorators.cache import cache_page
from . import views
urlpatterns = [
url(r'^settings/$', views.InstanceSettings.as_view(), name='settings'),
url(r'^stats/$',
cache_page(60 * 5)(views.InstanceStats.as_view()), name='stats'),
]
......@@ -4,6 +4,8 @@ from rest_framework.response import Response
from dynamic_preferences.api import serializers
from dynamic_preferences.registries import global_preferences_registry
from . import stats
class InstanceSettings(views.APIView):
permission_classes = []
......@@ -23,3 +25,12 @@ class InstanceSettings(views.APIView):
data = serializers.GlobalPreferenceSerializer(
api_preferences, many=True).data
return Response(data, status=200)
class InstanceStats(views.APIView):
permission_classes = []
authentication_classes = []
def get(self, request, *args, **kwargs):
data = stats.get()
return Response(data, status=200)
......@@ -25,13 +25,26 @@ class TrackAdmin(admin.ModelAdmin):
@admin.register(models.ImportBatch)
class ImportBatchAdmin(admin.ModelAdmin):
list_display = ['creation_date', 'status']
list_display = [
'submitted_by',
'creation_date',
'import_request',
'status']
list_select_related = [
'submitted_by',
'import_request',
]
list_filter = ['status']
search_fields = [
'import_request__name', 'source', 'batch__pk', 'mbid']
@admin.register(models.ImportJob)
class ImportJobAdmin(admin.ModelAdmin):
list_display = ['source', 'batch', 'track_file', 'status', 'mbid']
list_select_related = True
list_select_related = [
'track_file',
'batch',
]
search_fields = ['source', 'batch__pk', 'mbid']
list_filter = ['status']
......@@ -50,3 +63,19 @@ class LyricsAdmin(admin.ModelAdmin):
list_select_related = True
search_fields = ['url', 'work__title']
list_filter = ['work__language']
@admin.register(models.TrackFile)
class TrackFileAdmin(admin.ModelAdmin):
list_display = [
'track',
'audio_file',
'source',
'duration',
'mimetype',
]
list_select_related = [
'track'
]
search_fields = ['source', 'acoustid_track_id']
list_filter = ['mimetype']
......@@ -73,7 +73,10 @@ def _do_import(import_job, replace):
@celery.app.task(name='ImportJob.run', bind=True)
@celery.require_instance(models.ImportJob, 'import_job')
@celery.require_instance(
models.ImportJob.objects.filter(
status__in=['pending', 'errored']),
'import_job')
def import_job_run(self, import_job, replace=False):
def mark_errored():
import_job.status = 'errored'
......
......@@ -19,6 +19,7 @@ from musicbrainzngs import ResponseError
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from funkwhale_api.common import utils as funkwhale_utils
from funkwhale_api.requests.models import ImportRequest
from funkwhale_api.musicbrainz import api
from funkwhale_api.common.permissions import (
......@@ -62,7 +63,7 @@ class ArtistViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
'albums__tracks__tags'))
serializer_class = serializers.ArtistSerializerNested
permission_classes = [ConditionalAuthentication]
search_fields = ['name']
search_fields = ['name__unaccent']
filter_class = filters.ArtistFilter
ordering_fields = ('id', 'name', 'creation_date')
......@@ -75,7 +76,7 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
'tracks__files'))
serializer_class = serializers.AlbumSerializerNested
permission_classes = [ConditionalAuthentication]
search_fields = ['title']
search_fields = ['title__unaccent']
ordering_fields = ('creation_date',)
......@@ -116,7 +117,10 @@ class ImportJobViewSet(
def perform_create(self, serializer):
source = 'file://' + serializer.validated_data['audio_file'].name
serializer.save(source=source)
tasks.import_job_run.delay(import_job_id=serializer.instance.pk)
funkwhale_utils.on_commit(
tasks.import_job_run.delay,
import_job_id=serializer.instance.pk
)
class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
......@@ -129,9 +133,9 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
search_fields = ['title', 'artist__name']
ordering_fields = (
'creation_date',
'title',
'album__title',
'artist__name',
'title__unaccent',
'album__title__unaccent',
'artist__name__unaccent',
)
def get_queryset(self):
......@@ -245,7 +249,11 @@ class Search(views.APIView):
return Response(results, status=200)
def get_tracks(self, query):
search_fields = ['mbid', 'title', 'album__title', 'artist__name']
search_fields = [
'mbid',
'title__unaccent',
'album__title__unaccent',
'artist__name__unaccent']
query_obj = utils.get_query(query, search_fields)
return (
models.Track.objects.all()
......@@ -259,7 +267,10 @@ class Search(views.APIView):
def get_albums(self, query):
search_fields = ['mbid', 'title', 'artist__name']
search_fields = [
'mbid',
'title__unaccent',
'artist__name__unaccent']
query_obj = utils.get_query(query, search_fields)
return (
models.Album.objects.all()
......@@ -273,7 +284,7 @@ class Search(views.APIView):
def get_artists(self, query):
search_fields = ['mbid', 'name']
search_fields = ['mbid', 'name__unaccent']
query_obj = utils.get_query(query, search_fields)
return (
models.Artist.objects.all()
......@@ -288,7 +299,7 @@ class Search(views.APIView):
def get_tags(self, query):
search_fields = ['slug', 'name']
search_fields = ['slug', 'name__unaccent']
query_obj = utils.get_query(query, search_fields)
# We want the shortest tag first
......@@ -336,6 +347,7 @@ class SubmitViewSet(viewsets.ViewSet):
data, request, batch=None, import_request=import_request)
return Response(import_data)
@transaction.atomic
def _import_album(self, data, request, batch=None, import_request=None):
# we import the whole album here to prevent race conditions that occurs
# when using get_or_create_from_api in tasks
......@@ -355,7 +367,11 @@ class SubmitViewSet(viewsets.ViewSet):
models.TrackFile.objects.get(track__mbid=row['mbid'])
except models.TrackFile.DoesNotExist:
job = models.ImportJob.objects.create(mbid=row['mbid'], batch=batch, source=row['source'])
tasks.import_job_run.delay(import_job_id=job.pk)
funkwhale_utils.on_commit(
tasks.import_job_run.delay,
import_job_id=job.pk
)
serializer = serializers.ImportBatchSerializer(batch)
return serializer.data, batch
......
......@@ -3,6 +3,9 @@ import os
from django.core.files import File
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from funkwhale_api.common import utils
from funkwhale_api.music import tasks
from funkwhale_api.users.models import User
......@@ -86,6 +89,7 @@ class Command(BaseCommand):
self.stdout.write(
"For details, please refer to import batch #".format(batch.pk))
@transaction.atomic
def do_import(self, matching, user, options):
message = 'Importing {}...'
if options['async']:
......@@ -94,7 +98,7 @@ class Command(BaseCommand):
# we create an import batch binded to the user
batch = user.imports.create(source='shell')
async = options['async']
handler = tasks.import_job_run.delay if async else tasks.import_job_run
import_handler = tasks.import_job_run.delay if async else tasks.import_job_run
for path in matching:
job = batch.jobs.create(
source='file://' + path,
......@@ -105,7 +109,8 @@ class Command(BaseCommand):
job.save()
try:
handler(import_job_id=job.pk)
utils.on_commit(import_handler, import_job_id=job.pk)
except Exception as e:
self.stdout.write('Error: {}'.format(e))
return batch
from django.contrib import admin
from . import models
@admin.register(models.Radio)
class RadioAdmin(admin.ModelAdmin):
list_display = [
'user', 'name', 'is_public', 'creation_date', 'config']
list_select_related = [
'user',
]
list_filter = [
'is_public',
]
search_fields = ['name', 'description']
@admin.register(models.RadioSession)
class RadioSessionAdmin(admin.ModelAdmin):
list_display = [
'user',
'custom_radio',
'radio_type',
'creation_date',
'related_object']
list_select_related = [
'user',
'custom_radio'
]
list_filter = [
'radio_type',
]
@admin.register(models.RadioSessionTrack)
class RadioSessionTrackAdmin(admin.ModelAdmin):
list_display = [
'id',
'session',
'position',
'track',]
list_select_related = [
'track',
'session'
]
import random
from rest_framework import serializers
from django.db.models import Count
from django.core.exceptions import ValidationError
from taggit.models import Tag
from funkwhale_api.users.models import User
......@@ -39,8 +40,11 @@ class SessionRadio(SimpleRadio):
self.session = models.RadioSession.objects.create(user=user, radio_type=self.radio_type, **kwargs)
return self.session
def get_queryset(self):
raise NotImplementedError
def get_queryset(self, **kwargs):
qs = Track.objects.annotate(
files_count=Count('files')
)
return qs.filter(files_count__gt=0)
def get_queryset_kwargs(self):
return {}
......@@ -75,7 +79,9 @@ class SessionRadio(SimpleRadio):
@registry.register(name='random')
class RandomRadio(SessionRadio):
def get_queryset(self, **kwargs):
return Track.objects.all()
qs = super().get_queryset(**kwargs)
return qs.order_by('?')
@registry.register(name='favorites')
class FavoritesRadio(SessionRadio):
......@@ -87,8 +93,9 @@ class FavoritesRadio(SessionRadio):
return kwargs
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
track_ids = kwargs['user'].track_favorites.all().values_list('track', flat=True)
return Track.objects.filter(pk__in=track_ids)
return qs.filter(pk__in=track_ids)
@registry.register(name='custom')
......@@ -101,7 +108,11 @@ class CustomRadio(SessionRadio):
return kwargs
def get_queryset(self, **kwargs):
return filters.run(kwargs['custom_radio'].config)
qs = super().get_queryset(**kwargs)
return filters.run(
kwargs['custom_radio'].config,
candidates=qs,
)
def validate_session(self, data, **context):
data = super().validate_session(data, **context)
......@@ -141,6 +152,7 @@ class TagRadio(RelatedObjectRadio):
model = Tag
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
return Track.objects.filter(tags__in=[self.session.related_object])
@registry.register(name='artist')
......@@ -148,7 +160,8 @@ class ArtistRadio(RelatedObjectRadio):
model = Artist
def get_queryset(self, **kwargs):
return self.session.related_object.tracks.all()
qs = super().get_queryset(**kwargs)
return qs.filter(artist=self.session.related_object)
@registry.register(name='less-listened')
......@@ -160,5 +173,6 @@ class LessListenedRadio(RelatedObjectRadio):
super().clean(instance)
def get_queryset(self, **kwargs):
qs = super().get_queryset(**kwargs)
listened = self.session.user.listenings.all().values_list('track', flat=True)
return Track.objects.exclude(pk__in=listened).order_by('?')
return qs.exclude(pk__in=listened).order_by('?')
from django.contrib import admin
from . import models
@admin.register(models.ImportRequest)
class ImportRequestAdmin(admin.ModelAdmin):
list_display = ['artist_name', 'user', 'status', 'creation_date']
list_select_related = [
'user',
'track'
]
list_filter = [
'status',
]
search_fields = ['artist_name', 'comment', 'albums']
from django.urls import reverse
from funkwhale_api.instance import stats
def test_can_get_stats_via_api(db, api_client, mocker):
stats = {
'foo': 'bar'
}
mocker.patch('funkwhale_api.instance.stats.get', return_value=stats)
url = reverse('api:v1:instance:stats')
response = api_client.get(url)
assert response.data == stats
def test_get_users(mocker):
mocker.patch(
'funkwhale_api.users.models.User.objects.count', return_value=42)
assert stats.get_users() == 42
def test_get_music_duration(factories):
factories['music.TrackFile'].create_batch(size=5, duration=360)
# duration is in hours
assert stats.get_music_duration() == 0.5
def test_get_listenings(mocker):
mocker.patch(
'funkwhale_api.history.models.Listening.objects.count',
return_value=42)
assert stats.get_listenings() == 42
def test_get_track_favorites(mocker):
mocker.patch(
'funkwhale_api.favorites.models.TrackFavorite.objects.count',
return_value=42)
assert stats.get_track_favorites() == 42
def test_get_tracks(mocker):
mocker.patch(
'funkwhale_api.music.models.Track.objects.count',
return_value=42)
assert stats.get_tracks() == 42
def test_get_albums(mocker):
mocker.patch(
'funkwhale_api.music.models.Album.objects.count',
return_value=42)
assert stats.get_albums() == 42
def test_get_artists(mocker):
mocker.patch(
'funkwhale_api.music.models.Artist.objects.count',
return_value=42)
assert stats.get_artists() == 42
def test_get(mocker):
keys = [
'users',
'tracks',
'albums',
'artists',
'track_favorites',
'listenings',
'music_duration',
]
mocks = [
mocker.patch.object(stats, 'get_{}'.format(k), return_value=i)
for i, k in enumerate(keys)
]
expected = {
k: i for i, k in enumerate(keys)
}
assert stats.get() == expected
......@@ -6,6 +6,7 @@ from django.urls import reverse
from funkwhale_api.music import models
from funkwhale_api.musicbrainz import api
from funkwhale_api.music import serializers
from funkwhale_api.music import tasks
from . import data as api_data
......@@ -208,7 +209,7 @@ def test_user_can_create_an_empty_batch(client, factories):
def test_user_can_create_import_job_with_file(client, factories, mocker):
path = os.path.join(DATA_DIR, 'test.ogg')
m = mocker.patch('funkwhale_api.music.tasks.import_job_run.delay')
m = mocker.patch('funkwhale_api.common.utils.on_commit')
user = factories['users.SuperUser']()
batch = factories['music.ImportBatch'](submitted_by=user)
url = reverse('api:v1:import-jobs-list')
......@@ -231,7 +232,9 @@ def test_user_can_create_import_job_with_file(client, factories, mocker):
assert 'test.ogg' in job.source
assert job.audio_file.read() == content
m.assert_called_once_with(import_job_id=job.pk)
m.assert_called_once_with(
tasks.import_job_run.delay,
import_job_id=job.pk)
def test_can_search_artist(factories, client):
......
......@@ -51,7 +51,8 @@ def test_can_pick_by_weight():
def test_can_get_choices_for_favorites_radio(factories):
tracks = factories['music.Track'].create_batch(10)
files = factories['music.TrackFile'].create_batch(10)
tracks = [f.track for f in files]
user = factories['users.User']()
for i in range(5):
TrackFavorite.add(track=random.choice(tracks), user=user)
......@@ -71,8 +72,12 @@ def test_can_get_choices_for_favorites_radio(factories):
def test_can_get_choices_for_custom_radio(factories):
artist = factories['music.Artist']()
tracks = factories['music.Track'].create_batch(5, artist=artist)
wrong_tracks = factories['music.Track'].create_batch(5)
files = factories['music.TrackFile'].create_batch(
5, track__artist=artist)
tracks = [f.track for f in files]
wrong_files = factories['music.TrackFile'].create_batch(5)
wrong_tracks = [f.track for f in wrong_files]
session = factories['radios.CustomRadioSession'](
custom_radio__config=[{'type': 'artist', 'ids': [artist.pk]}]
)
......@@ -113,7 +118,8 @@ def test_can_start_custom_radio_from_api(logged_in_client, factories):
def test_can_use_radio_session_to_filter_choices(factories):
tracks = factories['music.Track'].create_batch(30)
files = factories['music.TrackFile'].create_batch(30)
tracks = [f.track for f in files]
user = factories['users.User']()
radio = radios.RandomRadio()
session = radio.start_session(user)
......@@ -156,8 +162,8 @@ def test_can_start_radio_for_anonymous_user(client, db):
def test_can_get_track_for_session_from_api(factories, logged_in_client):
tracks = factories['music.Track'].create_batch(size=1)
files = factories['music.TrackFile'].create_batch(1)
tracks = [f.track for f in files]
url = reverse('api:v1:radios:sessions-list')
response = logged_in_client.post(url, {'radio_type': 'random'})
session = models.RadioSession.objects.latest('id')
......@@ -169,7 +175,7 @@ def test_can_get_track_for_session_from_api(factories, logged_in_client):
assert data['track']['id'] == tracks[0].id
assert data['position'] == 1
next_track = factories['music.Track']()
next_track = factories['music.TrackFile']().track
response = logged_in_client.post(url, {'session': session.pk})
data = json.loads(response.content.decode('utf-8'))
......@@ -193,8 +199,11 @@ def test_related_object_radio_validate_related_object(factories):
def test_can_start_artist_radio(factories):
user = factories['users.User']()
artist = factories['music.Artist']()
wrong_tracks = factories['music.Track'].create_batch(5)
good_tracks = factories['music.Track'].create_batch(5, artist=artist)
wrong_files = factories['music.TrackFile'].create_batch(5)
wrong_tracks = [f.track for f in wrong_files]