Commit 08e28aa6 authored by Eliot Berriot's avatar Eliot Berriot

Merge branch 'release/0.3.1'

parents c8a2ae42 0b8f61b2
Changelog
=========
0.3.2 (Unreleased)
------------------
0.3.1
------------------
- Revamped all import logic, everything is more tested and consistend
- Can now use Acoustid in file imports to automatically grab metadata from musicbrainz
- Brand new file import wizard
0.2.7 (Unreleased)
0.2.7
------------------
- Shortcuts: can now use the ``f`` shortcut to toggle the currently playing track
......
......@@ -6,8 +6,8 @@ ENV PYTHONUNBUFFERED 1
COPY ./requirements.apt /requirements.apt
RUN apt-get update -qq && grep "^[^#;]" requirements.apt | xargs apt-get install -y
RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
RUN fcalc yolofkjdssdhf
COPY ./requirements/base.txt /requirements/base.txt
RUN pip install -r /requirements/base.txt
COPY ./requirements/production.txt /requirements/production.txt
......
......@@ -15,6 +15,7 @@ router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles')
router.register(r'artists', views.ArtistViewSet, 'artists')
router.register(r'albums', views.AlbumViewSet, 'albums')
router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches')
router.register(r'import-jobs', views.ImportJobViewSet, 'import-jobs')
router.register(r'submit', views.SubmitViewSet, 'submit')
router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists')
router.register(
......
......@@ -47,7 +47,6 @@ THIRD_PARTY_APPS = (
'corsheaders',
'rest_framework',
'rest_framework.authtoken',
'djcelery',
'taggit',
'cachalot',
'rest_auth',
......@@ -68,6 +67,7 @@ LOCAL_APPS = (
'funkwhale_api.playlists',
'funkwhale_api.providers.audiofile',
'funkwhale_api.providers.youtube',
'funkwhale_api.providers.acoustid',
)
# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
......@@ -266,14 +266,14 @@ CACHES["default"]["OPTIONS"] = {
########## CELERY
INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
BROKER_URL = env(
CELERY_BROKER_URL = env(
"CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT))
########## END CELERY
# Location of root django.contrib.admin URL, use {% url 'admin:index' %}
ADMIN_URL = r'^admin/'
# Your common stuff: Below this line define 3rd party library settings
CELERY_DEFAULT_RATE_LIMIT = 1
CELERYD_TASK_TIME_LIMIT = 300
CELERY_TASK_DEFAULT_RATE_LIMIT = 1
CELERY_TASK_TIME_LIMIT = 300
import datetime
JWT_AUTH = {
'JWT_ALLOW_REFRESH': True,
......
......@@ -54,7 +54,7 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
########## CELERY
# In development, all tasks will be executed locally by blocking until the task returns
CELERY_ALWAYS_EAGER = False
CELERY_TASK_ALWAYS_EAGER = False
########## END CELERY
# Your local stuff: Below this line define 3rd party library settings
......
......@@ -23,7 +23,7 @@ CACHES = {
}
}
BROKER_URL = 'memory://'
CELERY_BROKER_URL = 'memory://'
# TESTING
# ------------------------------------------------------------------------------
......@@ -31,7 +31,7 @@ TEST_RUNNER = 'django.test.runner.DiscoverRunner'
########## CELERY
# In development, all tasks will be executed locally by blocking until the task returns
CELERY_ALWAYS_EAGER = True
CELERY_TASK_ALWAYS_EAGER = True
########## END CELERY
# Your local stuff: Below this line define 3rd party library settings
......
......@@ -7,6 +7,7 @@ ENV PYTHONDONTWRITEBYTECODE 1
COPY ./requirements.apt /requirements.apt
COPY ./install_os_dependencies.sh /install_os_dependencies.sh
RUN bash install_os_dependencies.sh install
RUN curl -L https://github.com/acoustid/chromaprint/releases/download/v1.4.2/chromaprint-fpcalc-1.4.2-linux-x86_64.tar.gz | tar -xz -C /usr/local/bin --strip 1
RUN mkdir /requirements
COPY ./requirements/base.txt /requirements/base.txt
......
# -*- coding: utf-8 -*-
__version__ = '0.3'
__version__ = '0.3.1'
__version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])
from django.conf import settings
from rest_framework.permissions import BasePermission
from rest_framework.permissions import BasePermission, DjangoModelPermissions
class ConditionalAuthentication(BasePermission):
......@@ -9,3 +9,14 @@ class ConditionalAuthentication(BasePermission):
if settings.API_AUTHENTICATION_REQUIRED:
return request.user and request.user.is_authenticated
return True
class HasModelPermission(DjangoModelPermissions):
"""
Same as DjangoModelPermissions, but we pin the model:
class MyModelPermission(HasModelPermission):
model = User
"""
def get_required_permissions(self, method, model_cls):
return super().get_required_permissions(method, self.model)
......@@ -72,6 +72,14 @@ class ImportJobFactory(factory.django.DjangoModelFactory):
model = 'music.ImportJob'
@registry.register(name='music.FileImportJob')
class FileImportJobFactory(ImportJobFactory):
source = 'file://'
mbid = None
audio_file = factory.django.FileField(
from_path=os.path.join(SAMPLES_PATH, 'test.ogg'))
@registry.register
class WorkFactory(factory.django.DjangoModelFactory):
mbid = factory.Faker('uuid4')
......
# Generated by Django 2.0 on 2017-12-26 16:39
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('music', '0015_bind_track_file_to_import_job'),
]
operations = [
migrations.AddField(
model_name='trackfile',
name='acoustid_track_id',
field=models.UUIDField(blank=True, null=True),
),
]
# Generated by Django 2.0 on 2017-12-27 17:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('music', '0016_trackfile_acoustid_track_id'),
]
operations = [
migrations.AddField(
model_name='importbatch',
name='source',
field=models.CharField(choices=[('api', 'api'), ('shell', 'shell')], default='api', max_length=30),
),
migrations.AddField(
model_name='importjob',
name='audio_file',
field=models.FileField(blank=True, max_length=255, null=True, upload_to='imports/%Y/%m/%d'),
),
migrations.AlterField(
model_name='importjob',
name='mbid',
field=models.UUIDField(blank=True, editable=False, null=True),
),
]
......@@ -15,11 +15,9 @@ from django.utils import timezone
from taggit.managers import TaggableManager
from versatileimagefield.fields import VersatileImageField
from funkwhale_api.taskapp import celery
from funkwhale_api import downloader
from funkwhale_api import musicbrainz
from . import importers
from . import lyrics as lyrics_utils
class APIModelMixin(models.Model):
......@@ -255,14 +253,6 @@ class Lyrics(models.Model):
url = models.URLField(unique=True)
content = models.TextField(null=True, blank=True)
@celery.app.task(name='Lyrics.fetch_content', filter=celery.task_method)
def fetch_content(self):
html = lyrics_utils._get_html(self.url)
content = lyrics_utils.extract_content(html)
cleaned_content = lyrics_utils.clean_content(content)
self.content = cleaned_content
self.save()
@property
def content_rendered(self):
return markdown.markdown(
......@@ -362,6 +352,7 @@ class TrackFile(models.Model):
audio_file = models.FileField(upload_to='tracks/%Y/%m/%d', max_length=255)
source = models.URLField(null=True, blank=True)
duration = models.IntegerField(null=True, blank=True)
acoustid_track_id = models.UUIDField(null=True, blank=True)
def download_file(self):
# import the track file, since there is not any
......@@ -393,9 +384,17 @@ class TrackFile(models.Model):
class ImportBatch(models.Model):
IMPORT_BATCH_SOURCES = [
('api', 'api'),
('shell', 'shell')
]
source = models.CharField(
max_length=30, default='api', choices=IMPORT_BATCH_SOURCES)
creation_date = models.DateTimeField(default=timezone.now)
submitted_by = models.ForeignKey(
'users.User', related_name='imports', on_delete=models.CASCADE)
'users.User',
related_name='imports',
on_delete=models.CASCADE)
class Meta:
ordering = ['-creation_date']
......@@ -406,8 +405,11 @@ class ImportBatch(models.Model):
@property
def status(self):
pending = any([job.status == 'pending' for job in self.jobs.all()])
errored = any([job.status == 'errored' for job in self.jobs.all()])
if pending:
return 'pending'
if errored:
return 'errored'
return 'finished'
class ImportJob(models.Model):
......@@ -419,36 +421,17 @@ class ImportJob(models.Model):
null=True,
blank=True,
on_delete=models.CASCADE)
source = models.URLField()
mbid = models.UUIDField(editable=False)
source = models.CharField(max_length=500)
mbid = models.UUIDField(editable=False, null=True, blank=True)
STATUS_CHOICES = (
('pending', 'Pending'),
('finished', 'finished'),
('finished', 'Finished'),
('errored', 'Errored'),
('skipped', 'Skipped'),
)
status = models.CharField(choices=STATUS_CHOICES, default='pending', max_length=30)
audio_file = models.FileField(
upload_to='imports/%Y/%m/%d', max_length=255, null=True, blank=True)
class Meta:
ordering = ('id', )
@celery.app.task(name='ImportJob.run', filter=celery.task_method)
def run(self, replace=False):
try:
track, created = Track.get_or_create_from_api(mbid=self.mbid)
track_file = None
if replace:
track_file = track.files.first()
elif track.files.count() > 0:
return
track_file = track_file or TrackFile(
track=track, source=self.source)
track_file.download_file()
track_file.save()
self.status = 'finished'
self.track_file = track_file
self.save()
return track.pk
except Exception as exc:
if not settings.DEBUG:
raise ImportJob.run.retry(args=[self], exc=exc, countdown=30, max_retries=3)
raise
......@@ -113,7 +113,8 @@ class ImportJobSerializer(serializers.ModelSerializer):
track_file = TrackFileSerializer(read_only=True)
class Meta:
model = models.ImportJob
fields = ('id', 'mbid', 'source', 'status', 'track_file')
fields = ('id', 'mbid', 'batch', 'source', 'status', 'track_file', 'audio_file')
read_only_fields = ('status', 'track_file')
class ImportBatchSerializer(serializers.ModelSerializer):
......@@ -121,3 +122,4 @@ class ImportBatchSerializer(serializers.ModelSerializer):
class Meta:
model = models.ImportBatch
fields = ('id', 'jobs', 'status', 'creation_date')
read_only_fields = ('creation_date',)
from django.core.files.base import ContentFile
from funkwhale_api.taskapp import celery
from funkwhale_api.providers.acoustid import get_acoustid_client
from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
from django.conf import settings
from . import models
from . import lyrics as lyrics_utils
@celery.app.task(name='acoustid.set_on_track_file')
@celery.require_instance(models.TrackFile, 'track_file')
def set_acoustid_on_track_file(track_file):
client = get_acoustid_client()
result = client.get_best_match(track_file.audio_file.path)
def update(id):
track_file.acoustid_track_id = id
track_file.save(update_fields=['acoustid_track_id'])
return id
if result:
return update(result['id'])
def _do_import(import_job, replace):
from_file = bool(import_job.audio_file)
mbid = import_job.mbid
acoustid_track_id = None
duration = None
track = None
if not mbid and from_file:
# we try to deduce mbid from acoustid
client = get_acoustid_client()
match = client.get_best_match(import_job.audio_file.path)
if not match:
raise ValueError('Cannot get match')
duration = match['recordings'][0]['duration']
mbid = match['recordings'][0]['id']
acoustid_track_id = match['id']
if mbid:
track, _ = models.Track.get_or_create_from_api(mbid=mbid)
else:
track = import_track_data_from_path(import_job.audio_file.path)
track_file = None
if replace:
track_file = track.files.first()
elif track.files.count() > 0:
if import_job.audio_file:
import_job.audio_file.delete()
import_job.status = 'skipped'
import_job.save()
return
track_file = track_file or models.TrackFile(
track=track, source=import_job.source)
track_file.acoustid_track_id = acoustid_track_id
if from_file:
track_file.audio_file = ContentFile(import_job.audio_file.read())
track_file.audio_file.name = import_job.audio_file.name
track_file.duration = duration
else:
track_file.download_file()
track_file.save()
import_job.status = 'finished'
import_job.track_file = track_file
if import_job.audio_file:
# it's imported on the track, we don't need it anymore
import_job.audio_file.delete()
import_job.save()
return track.pk
@celery.app.task(name='ImportJob.run', bind=True)
@celery.require_instance(models.ImportJob, 'import_job')
def import_job_run(self, import_job, replace=False):
def mark_errored():
import_job.status = 'errored'
import_job.save()
try:
return _do_import(import_job, replace)
except Exception as exc:
if not settings.DEBUG:
try:
self.retry(exc=exc, countdown=30, max_retries=3)
except:
mark_errored()
raise
mark_errored()
raise
@celery.app.task(name='Lyrics.fetch_content')
@celery.require_instance(models.Lyrics, 'lyrics')
def fetch_content(lyrics):
html = lyrics_utils._get_html(lyrics.url)
content = lyrics_utils.extract_content(html)
cleaned_content = lyrics_utils.clean_content(content)
lyrics.content = cleaned_content
lyrics.save(update_fields=['content'])
......@@ -6,7 +6,7 @@ from django.urls import reverse
from django.db import models, transaction
from django.db.models.functions import Length
from django.conf import settings
from rest_framework import viewsets, views
from rest_framework import viewsets, views, mixins
from rest_framework.decorators import detail_route, list_route
from rest_framework.response import Response
from rest_framework import permissions
......@@ -15,13 +15,15 @@ from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from funkwhale_api.musicbrainz import api
from funkwhale_api.common.permissions import ConditionalAuthentication
from funkwhale_api.common.permissions import (
ConditionalAuthentication, HasModelPermission)
from taggit.models import Tag
from . import models
from . import serializers
from . import importers
from . import filters
from . import tasks
from . import utils
......@@ -70,16 +72,45 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
ordering_fields = ('creation_date',)
class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
class ImportBatchViewSet(
mixins.CreateModelMixin,
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
viewsets.GenericViewSet):
queryset = (
models.ImportBatch.objects.all()
.prefetch_related('jobs__track_file')
.order_by('-creation_date'))
serializer_class = serializers.ImportBatchSerializer
permission_classes = (permissions.DjangoModelPermissions, )
def get_queryset(self):
return super().get_queryset().filter(submitted_by=self.request.user)
def perform_create(self, serializer):
serializer.save(submitted_by=self.request.user)
class ImportJobPermission(HasModelPermission):
# not a typo, perms on import job is proxied to import batch
model = models.ImportBatch
class ImportJobViewSet(
mixins.CreateModelMixin,
viewsets.GenericViewSet):
queryset = (models.ImportJob.objects.all())
serializer_class = serializers.ImportJobSerializer
permission_classes = (ImportJobPermission, )
def get_queryset(self):
return super().get_queryset().filter(batch__submitted_by=self.request.user)
def perform_create(self, serializer):
source = 'file://' + serializer.validated_data['audio_file'].name
serializer.save(source=source)
tasks.import_job_run.delay(import_job_id=serializer.instance.pk)
class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
"""
......@@ -129,7 +160,8 @@ class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
lyrics = work.fetch_lyrics()
try:
if not lyrics.content:
lyrics.fetch_content()
tasks.fetch_content(lyrics_id=lyrics.pk)
lyrics.refresh_from_db()
except AttributeError:
return Response({'error': 'unavailable lyrics'}, status=404)
serializer = serializers.LyricsSerializer(lyrics)
......@@ -244,7 +276,7 @@ class SubmitViewSet(viewsets.ViewSet):
pass
batch = models.ImportBatch.objects.create(submitted_by=request.user)
job = models.ImportJob.objects.create(mbid=request.POST['mbid'], batch=batch, source=request.POST['import_url'])
job.run.delay()
tasks.import_job_run.delay(import_job_id=job.pk)
serializer = serializers.ImportBatchSerializer(batch)
return Response(serializer.data)
......@@ -272,7 +304,7 @@ class SubmitViewSet(viewsets.ViewSet):
models.TrackFile.objects.get(track__mbid=row['mbid'])
except models.TrackFile.DoesNotExist:
job = models.ImportJob.objects.create(mbid=row['mbid'], batch=batch, source=row['source'])
job.run.delay()
tasks.import_job_run.delay(import_job_id=job.pk)
serializer = serializers.ImportBatchSerializer(batch)
return serializer.data, batch
......
import acoustid
from dynamic_preferences.registries import global_preferences_registry
class Client(object):
def __init__(self, api_key):
self.api_key = api_key
def match(self, file_path):
return acoustid.match(self.api_key, file_path, parse=False)
def get_best_match(self, file_path):
results = self.match(file_path=file_path)
MIN_SCORE_FOR_MATCH = 0.8
try:
rows = results['results']
except KeyError:
return
for row in rows:
if row['score'] >= MIN_SCORE_FOR_MATCH:
return row
def get_acoustid_client():
manager = global_preferences_registry.manager()
return Client(api_key=manager['providers_acoustid__api_key'])
from dynamic_preferences.types import StringPreference, Section
from dynamic_preferences.registries import global_preferences_registry
acoustid = Section('providers_acoustid')
@global_preferences_registry.register
class APIKey(StringPreference):
section = acoustid
name = 'api_key'
default = ''
verbose_name = 'Acoustid API key'
help_text = 'The API key used to query AcoustID. Get one at https://acoustid.org/new-application.'
import glob
import os
from django.core.files import File
from django.core.management.base import BaseCommand, CommandError
from funkwhale_api.providers.audiofile import tasks
from funkwhale_api.music import tasks
from funkwhale_api.users.models import User
class Command(BaseCommand):
......@@ -15,6 +19,11 @@ class Command(BaseCommand):
default=False,
help='Will match the pattern recursively (including subdirectories)',
)
parser.add_argument(
'--username',
dest='username',
help='The username of the user you want to be bound to the import',
)
parser.add_argument(
'--async',
action='store_true',
......@@ -46,6 +55,20 @@ class Command(BaseCommand):
if not matching:
raise CommandError('No file matching pattern, aborting')
user = None
if options['username']:
try:
user = User.objects.get(username=options['username'])
except User.DoesNotExist:
raise CommandError('Invalid username')
else:
# we bind the import to the first registered superuser
try:
user = User.objects.filter(is_superuser=True).order_by('pk').first()
assert user is not None
except AssertionError:
raise CommandError(
'No superuser available, please provide a --username')
if options['interactive']:
message = (
'Are you sure you want to do this?\n\n'
......@@ -54,18 +77,35 @@ class Command(BaseCommand):
if input(''.join(message)) != 'yes':
raise CommandError("Import cancelled.")
batch = self.do_import(matching, user=user, options=options)
message = 'Successfully imported {} tracks'
if options['async']:
message = 'Successfully launched import for {} tracks'
self.stdout.write(message.format(len(matching)))
self.stdout.write(
"For details, please refer to import batch #".format(batch.pk))
def do_import(self, matching, user, options):
message = 'Importing {}...'
if options['async']:
message = 'Launching import for {}...'
# we create an import batch binded to the user
batch = user.imports.create(source='shell')
async = options['async']
handler = tasks.import_job_run.delay if async else tasks.import_job_run
for path in matching:
self.stdout.write(message.format(path))
job = batch.jobs.create(
source='file://' + path,
)
name = os.path.basename(path)
with open(path, 'rb') as f:
job.audio_file.save(name, File(f))
job.save()
try:
tasks.from_path(path)
handler(import_job_id=job.pk)
except Exception as e:
self.stdout.write('Error: {}'.format(e))
message = 'Successfully imported {} tracks'
if options['async']:
message = 'Successfully launched import for {} tracks'
self.stdout.write(message.format(len(matching)))
return batch
import acoustid
import os
import datetime
from django.core.files import File
from funkwhale_api.taskapp import celery
from funkwhale_api.providers.acoustid import get_acoustid_client
from funkwhale_api.music import models, metadata
@celery.app.task(name='audiofile.from_path')
def from_path(path):
def import_track_data_from_path(path):
data = metadata.Metadata(path)
artist = models.Artist.objects.get_or_create(
name__iexact=data.get('artist'),
defaults={
'name': data.get('artist'),
'mbid': data.get('musicbrainz_artistid', None),
},
)[0]
......@@ -39,11 +39,33 @@ def from_path(path):
'mbid': data.get('musicbrainz_recordingid', None),
},
)[0]
return track
def import_metadata_with_musicbrainz(path):
pass
@celery.app.task(name='audiofile.from_path')
def from_path(path):
acoustid_track_id = None
try:
client = get_acoustid_client()
result = client.get_best_match(path)
acoustid_track_id = result['id']