Commit 142a8050 authored by Eliot Berriot's avatar Eliot Berriot

Merge branch 'release/0.2.4'

parents e9a3c37a 9d81ece0
......@@ -18,6 +18,8 @@ test_api:
- pip install -r requirements/test.txt
script:
- pytest
variables:
DATABASE_URL: "sqlite://"
tags:
- docker
......
Changelog
=========
0.2.5 (unreleased)
------------------
0.2.4 (2017-12-14)
------------------
Features:
- Models: now store relese group mbid on Album model (#7)
- Models: now bind import job to track files (#44)
Bugfixes:
- Library: fixen broken "play all albums" button on artist cards in Artist browsing view (#43)
......@@ -4,7 +4,7 @@ set -e
# Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple
# environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint
# does all this for us.
export REDIS_URL=redis://redis:6379/0
export CACHE_URL=redis://redis:6379/0
# the official postgres image uses 'postgres' as default user if not set explictly.
if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then
......@@ -13,7 +13,7 @@ fi
export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER
export CELERY_BROKER_URL=$REDIS_URL
export CELERY_BROKER_URL=$CACHE_URL
# we copy the frontend files, if any so we can serve them from the outside
if [ -d "frontend" ]; then
......
......@@ -124,7 +124,7 @@ MANAGERS = ADMINS
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
'default': env.db("DATABASE_URL", default="postgresql://postgres@postgres/postgres"),
'default': env.db("DATABASE_URL"),
}
DATABASES['default']['ATOMIC_REQUESTS'] = True
#
......@@ -199,7 +199,7 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3'
# STATIC FILE CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT = str(ROOT_DIR('staticfiles'))
STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles')))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = env("STATIC_URL", default='/staticfiles/')
......@@ -218,12 +218,10 @@ STATICFILES_FINDERS = (
# MEDIA CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = str(APPS_DIR('media'))
MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR('media')))
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = '/media/'
MEDIA_URL = env("MEDIA_URL", default='/media/')
# URL Configuration
# ------------------------------------------------------------------------------
......@@ -253,26 +251,24 @@ LOGIN_URL = 'account_login'
# SLUGLIFIER
AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify'
########## CELERY
INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
# if you are not using the django database broker (e.g. rabbitmq, redis, memcached), you can remove the next line.
INSTALLED_APPS += ('kombu.transport.django',)
BROKER_URL = env("CELERY_BROKER_URL", default='django://')
########## END CELERY
CACHE_DEFAULT = "redis://127.0.0.1:6379/0"
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "{0}/{1}".format(env.cache_url('REDIS_URL', default="redis://127.0.0.1:6379"), 0),
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"IGNORE_EXCEPTIONS": True, # mimics memcache behavior.
# http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
}
}
"default": env.cache_url('CACHE_URL', default=CACHE_DEFAULT)
}
CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
CACHES["default"]["OPTIONS"] = {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"IGNORE_EXCEPTIONS": True, # mimics memcache behavior.
# http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
}
########## CELERY
INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',)
BROKER_URL = env(
"CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT))
########## END CELERY
# Location of root django.contrib.admin URL, use {% url 'admin:index' %}
ADMIN_URL = r'^admin/'
# Your common stuff: Below this line define 3rd party library settings
......@@ -336,3 +332,8 @@ MUSICBRAINZ_CACHE_DURATION = env.int(
)
CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True)
# Custom Admin URL, use {% url 'admin:index' %}
ADMIN_URL = env('DJANGO_ADMIN_URL', default='^api/admin/')
CSRF_USE_SESSIONS = True
......@@ -54,7 +54,7 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
# ------------------------------------------------------------------------------
# Hosts/domain names that are valid for this site
# See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts
ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['funkwhale.io'])
ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS')
# END SITE CONFIGURATION
INSTALLED_APPS += ("gunicorn", )
......@@ -65,10 +65,6 @@ INSTALLED_APPS += ("gunicorn", )
# ------------------------
DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
# URL that handles the media served from MEDIA_ROOT, used for managing
# stored files.
MEDIA_URL = '/media/'
# Static Assets
# ------------------------
STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'
......@@ -92,11 +88,6 @@ TEMPLATES[0]['OPTIONS']['loaders'] = [
'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ]),
]
# DATABASE CONFIGURATION
# ------------------------------------------------------------------------------
# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
DATABASES['default'] = env.db("DATABASE_URL")
# CACHING
# ------------------------------------------------------------------------------
# Heroku URL does not pass the DB number, so we parse it in
......@@ -151,7 +142,5 @@ LOGGING = {
}
}
# Custom Admin URL, use {% url 'admin:index' %}
ADMIN_URL = env('DJANGO_ADMIN_URL')
# Your production stuff: Below this line define 3rd party library settings
......@@ -22,6 +22,9 @@ CACHES = {
'LOCATION': ''
}
}
INSTALLED_APPS += ('kombu.transport.django',)
BROKER_URL = 'django://'
# TESTING
# ------------------------------------------------------------------------------
TEST_RUNNER = 'django.test.runner.DiscoverRunner'
......
# -*- coding: utf-8 -*-
__version__ = '0.2.3'
__version__ = '0.2.4'
__version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace('-', '.', 1).split('.')])
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2017-12-14 22:05
from __future__ import unicode_literals
import django.contrib.sites.models
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sites', '0002_set_site_domain_and_name'),
]
operations = [
migrations.AlterModelManagers(
name='site',
managers=[
('objects', django.contrib.sites.models.SiteManager()),
],
),
migrations.AlterField(
model_name='site',
name='domain',
field=models.CharField(max_length=100, unique=True, validators=[django.contrib.sites.models._simple_domain_name_validator], verbose_name='domain name'),
),
]
......@@ -2,30 +2,35 @@ from django.contrib import admin
from . import models
@admin.register(models.Artist)
class ArtistAdmin(admin.ModelAdmin):
list_display = ['name', 'mbid', 'creation_date']
search_fields = ['name', 'mbid']
@admin.register(models.Album)
class AlbumAdmin(admin.ModelAdmin):
list_display = ['title', 'artist', 'mbid', 'release_date', 'creation_date']
search_fields = ['title', 'artist__name', 'mbid']
list_select_related = True
@admin.register(models.Track)
class TrackAdmin(admin.ModelAdmin):
list_display = ['title', 'artist', 'album', 'mbid']
search_fields = ['title', 'artist__name', 'album__title', 'mbid']
list_select_related = True
@admin.register(models.ImportBatch)
class ImportBatchAdmin(admin.ModelAdmin):
list_display = ['creation_date', 'status']
@admin.register(models.ImportJob)
class ImportJobAdmin(admin.ModelAdmin):
list_display = ['source', 'batch', 'status', 'mbid']
list_display = ['source', 'batch', 'track_file', 'status', 'mbid']
list_select_related = True
search_fields = ['source', 'batch__pk', 'mbid']
list_filter = ['status']
......
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2017-12-13 22:11
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('music', '0012_auto_20161122_1905'),
]
operations = [
migrations.AlterModelOptions(
name='importjob',
options={'ordering': ('id',)},
),
migrations.AlterModelOptions(
name='track',
options={'ordering': ['album', 'position']},
),
migrations.AddField(
model_name='album',
name='release_group_id',
field=models.UUIDField(blank=True, null=True),
),
]
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2017-12-14 21:14
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('music', '0013_auto_20171213_2211'),
]
operations = [
migrations.AddField(
model_name='importjob',
name='track_file',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='jobs', to='music.TrackFile'),
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os
from django.db import migrations, models
from funkwhale_api.common.utils import rename_file
def bind_jobs(apps, schema_editor):
TrackFile = apps.get_model("music", "TrackFile")
ImportJob = apps.get_model("music", "ImportJob")
for job in ImportJob.objects.all().only('mbid'):
f = TrackFile.objects.filter(track__mbid=job.mbid).first()
if not f:
print('No file for mbid {}'.format(job.mbid))
continue
job.track_file = f
job.save(update_fields=['track_file'])
def rewind(apps, schema_editor):
pass
class Migration(migrations.Migration):
dependencies = [
('music', '0014_importjob_track_file'),
]
operations = [
migrations.RunPython(bind_jobs, rewind),
]
......@@ -110,13 +110,14 @@ class Album(APIModelMixin):
title = models.CharField(max_length=255)
artist = models.ForeignKey(Artist, related_name='albums')
release_date = models.DateField(null=True)
release_group_id = models.UUIDField(null=True, blank=True)
cover = VersatileImageField(upload_to='albums/covers/%Y/%m/%d', null=True, blank=True)
TYPE_CHOICES = (
('album', 'Album'),
)
type = models.CharField(choices=TYPE_CHOICES, max_length=30, default='album')
api_includes = ['artist-credits', 'recordings', 'media']
api_includes = ['artist-credits', 'recordings', 'media', 'release-groups']
api = musicbrainz.api.releases
musicbrainz_model = 'release'
musicbrainz_mapping = {
......@@ -127,6 +128,10 @@ class Album(APIModelMixin):
'musicbrainz_field_name': 'release-list',
'converter': lambda v: int(v[0]['medium-list'][0]['position']),
},
'release_group_id': {
'musicbrainz_field_name': 'release-group',
'converter': lambda v: v['id'],
},
'title': {
'musicbrainz_field_name': 'title',
},
......@@ -388,6 +393,8 @@ class ImportBatch(models.Model):
class ImportJob(models.Model):
batch = models.ForeignKey(ImportBatch, related_name='jobs')
track_file = models.ForeignKey(
TrackFile, related_name='jobs', null=True, blank=True)
source = models.URLField()
mbid = models.UUIDField(editable=False)
STATUS_CHOICES = (
......@@ -408,10 +415,12 @@ class ImportJob(models.Model):
elif track.files.count() > 0:
return
track_file = track_file or TrackFile(track=track, source=self.source)
track_file = track_file or TrackFile(
track=track, source=self.source)
track_file.download_file()
track_file.save()
self.status = 'finished'
self.track_file = track_file
self.save()
return track.pk
......
......@@ -9,35 +9,26 @@ class TagSerializer(serializers.ModelSerializer):
model = Tag
fields = ('id', 'name', 'slug')
class SimpleArtistSerializer(serializers.ModelSerializer):
class Meta:
model = models.Artist
fields = ('id', 'mbid', 'name')
class ArtistSerializer(serializers.ModelSerializer):
tags = TagSerializer(many=True, read_only=True)
class Meta:
model = models.Artist
fields = ('id', 'mbid', 'name', 'tags')
class ImportJobSerializer(serializers.ModelSerializer):
class Meta:
model = models.ImportJob
fields = ('id', 'mbid', 'source', 'status')
class ImportBatchSerializer(serializers.ModelSerializer):
jobs = ImportJobSerializer(many=True, read_only=True)
class Meta:
model = models.ImportBatch
fields = ('id', 'jobs', 'status', 'creation_date')
class TrackFileSerializer(serializers.ModelSerializer):
path = serializers.SerializerMethodField()
class Meta:
model = models.TrackFile
fields = ('id', 'path', 'duration', 'source', 'filename')
fields = ('id', 'path', 'duration', 'source', 'filename', 'track')
def get_path(self, o):
request = self.context.get('request')
......@@ -46,12 +37,14 @@ class TrackFileSerializer(serializers.ModelSerializer):
url = request.build_absolute_uri(url)
return url
class SimpleAlbumSerializer(serializers.ModelSerializer):
class Meta:
model = models.Album
fields = ('id', 'mbid', 'title', 'release_date', 'cover')
class AlbumSerializer(serializers.ModelSerializer):
tags = TagSerializer(many=True, read_only=True)
class Meta:
......@@ -81,6 +74,7 @@ class TrackSerializer(LyricsMixin):
'position',
'lyrics')
class TrackSerializerNested(LyricsMixin):
artist = ArtistSerializer()
files = TrackFileSerializer(many=True, read_only=True)
......@@ -90,6 +84,7 @@ class TrackSerializerNested(LyricsMixin):
model = models.Track
fields = ('id', 'mbid', 'title', 'artist', 'files', 'album', 'tags', 'lyrics')
class AlbumSerializerNested(serializers.ModelSerializer):
tracks = TrackSerializer(many=True, read_only=True)
artist = SimpleArtistSerializer()
......@@ -99,6 +94,7 @@ class AlbumSerializerNested(serializers.ModelSerializer):
model = models.Album
fields = ('id', 'mbid', 'title', 'cover', 'artist', 'release_date', 'tracks', 'tags')
class ArtistSerializerNested(serializers.ModelSerializer):
albums = AlbumSerializerNested(many=True, read_only=True)
tags = TagSerializer(many=True, read_only=True)
......@@ -111,3 +107,17 @@ class LyricsSerializer(serializers.ModelSerializer):
class Meta:
model = models.Lyrics
fields = ('id', 'work', 'content', 'content_rendered')
class ImportJobSerializer(serializers.ModelSerializer):
track_file = TrackFileSerializer(read_only=True)
class Meta:
model = models.ImportJob
fields = ('id', 'mbid', 'source', 'status', 'track_file')
class ImportBatchSerializer(serializers.ModelSerializer):
jobs = ImportJobSerializer(many=True, read_only=True)
class Meta:
model = models.ImportBatch
fields = ('id', 'jobs', 'status', 'creation_date')
import factory
import os
from funkwhale_api.users.tests.factories import UserFactory
SAMPLES_PATH = os.path.dirname(os.path.abspath(__file__))
......@@ -18,6 +20,7 @@ class AlbumFactory(factory.django.DjangoModelFactory):
release_date = factory.Faker('date')
cover = factory.django.ImageField()
artist = factory.SubFactory(ArtistFactory)
release_group_id = factory.Faker('uuid4')
class Meta:
model = 'music.Album'
......@@ -41,3 +44,18 @@ class TrackFileFactory(factory.django.DjangoModelFactory):
class Meta:
model = 'music.TrackFile'
class ImportBatchFactory(factory.django.DjangoModelFactory):
submitted_by = factory.SubFactory(UserFactory)
class Meta:
model = 'music.ImportBatch'
class ImportJobFactory(factory.django.DjangoModelFactory):
batch = factory.SubFactory(ImportBatchFactory)
source = factory.Faker('url')
class Meta:
model = 'music.ImportJob'
import pytest
from funkwhale_api.music import models
from funkwhale_api.music import importers
from . import factories
def test_can_store_release_group_id_on_album(db):
album = factories.AlbumFactory()
assert album.release_group_id is not None
def test_import_album_stores_release_group(db):
album_data = {
"artist-credit": [
{
"artist": {
"disambiguation": "George Shaw",
"id": "62c3befb-6366-4585-b256-809472333801",
"name": "Adhesive Wombat",
"sort-name": "Wombat, Adhesive"
}
}
],
"artist-credit-phrase": "Adhesive Wombat",
"country": "XW",
"date": "2013-06-05",
"id": "a50d2a81-2a50-484d-9cb4-b9f6833f583e",
"status": "Official",
"title": "Marsupial Madness",
'release-group': {'id': '447b4979-2178-405c-bfe6-46bf0b09e6c7'}
}
artist = factories.ArtistFactory(
mbid=album_data['artist-credit'][0]['artist']['id']
)
cleaned_data = models.Album.clean_musicbrainz_data(album_data)
album = importers.load(models.Album, cleaned_data, album_data, import_hooks=[])
assert album.release_group_id == album_data['release-group']['id']
assert album.artist == artist
def test_import_job_is_bound_to_track_file(db, mocker):
track = factories.TrackFactory()
job = factories.ImportJobFactory(mbid=track.mbid)
mocker.patch('funkwhale_api.music.models.TrackFile.download_file')
job.run()
job.refresh_from_db()
assert job.track_file.track == track
......@@ -72,7 +72,10 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
queryset = models.ImportBatch.objects.all().order_by('-creation_date')
queryset = (
models.ImportBatch.objects.all()
.prefetch_related('jobs__track_file')
.order_by('-creation_date'))
serializer_class = serializers.ImportBatchSerializer
def get_queryset(self):
......
import glob
from django.core.management.base import BaseCommand, CommandError
from funkwhale_api.providers.audiofile import importer
from funkwhale_api.providers.audiofile import tasks
class Command(BaseCommand):
......@@ -61,7 +61,7 @@ class Command(BaseCommand):
for path in matching:
self.stdout.write(message.format(path))
try:
importer.from_path(path)
tasks.from_path(path)
except Exception as e:
self.stdout.write('Error: {}'.format(e))
......
......@@ -3,7 +3,7 @@ import datetime
import unittest
from test_plus.test import TestCase
from funkwhale_api.providers.audiofile import importer
from funkwhale_api.providers.audiofile import tasks
DATA_DIR = os.path.dirname(os.path.abspath(__file__))
......@@ -27,7 +27,7 @@ class TestAudioFile(TestCase):
return_value='OggVorbis',
)
with m1, m2:
track_file = importer.from_path(
track_file = tasks.from_path(
os.path.join(DATA_DIR, 'dummy_file.ogg'))
self.assertEqual(
......
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2017-12-14 22:05
from __future__ import unicode_literals
import django.contrib.auth.models
import django.contrib.auth.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('users', '0001_initial'),
]
operations = [
migrations.AlterModelManagers(
name='user',
managers=[
('objects', django.contrib.auth.models.UserManager()),
],
),
migrations.AlterField(
model_name='user',
name='username',
field=models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username'),
),
]
......@@ -2,8 +2,11 @@
import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production")
from django.core.management import execute_from_command_line
......
##basic build dependencies of various Django apps for Ubuntu 14.04
#build-essential metapackage install: make, gcc, g++,
build-essential
#required to translate
gettext
#python-dev
##shared dependencies of:
##Pillow, pylibmc
zlib1g-dev
##Postgresql and psycopg2 dependencies
libjpeg-dev
zlib1g-dev
libpq-dev
postgresql-client
##Pillow dependencies
#libtiff4-dev
#libjpeg8-dev
#libfreetype6-dev
#liblcms1-dev
#libwebp-dev
##django-extensions
#graphviz-dev
##hitch
#python-setuptools
#python3-dev
#python-virtualenv
#python-pip
#firefox
#automake
#libtool
#libreadline6
#libreadline6-dev
#libreadline-dev
libsqlite3-dev
#libxml2
#libxml2-dev
#libssl-dev
#libbz2-dev
#wget
#curl
#llvm
libav-tools
python3-dev