Commit 46a385a7 authored by Eliot Berriot's avatar Eliot Berriot

WIP

parent 66acb3fd
from . import create_actors
from . import create_image_variations
from . import django_permissions_to_user_permissions
from . import migrate_to_user_libraries
from . import test
......@@ -8,5 +9,6 @@ __all__ = [
"create_actors",
"create_image_variations",
"django_permissions_to_user_permissions",
"migrate_to_user_libraries",
"test",
]
"""
Mirate instance files to a library #463. For each user that imported music on an
instance, we will create a "default" library with related files and an instance-level
visibility.
Files without any import job will be bounded to a "default" library on the first
superuser account found. This should now happen though.
"""
from funkwhale_api.music import models
from funkwhale_api.users.models import User
def main(command, **kwargs):
importer_ids = set(
models.ImportBatch.objects.values_list("submitted_by", flat=True)
)
importers = User.objects.filter(pk__in=importer_ids).order_by("id").select_related()
command.stdout.write(
"* {} users imported music on this instance".format(len(importers))
)
files = models.TrackFile.objects.filter(
library__isnull=True, jobs__isnull=False
).distinct()
command.stdout.write(
"* Reassigning {} files to importers libraries...".format(files.count())
)
for user in importers:
command.stdout.write(
" * Setting up @{}'s 'default' library".format(user.username)
)
library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[
0
]
user_files = files.filter(jobs__batch__submitted_by=user)
total = user_files.count()
command.stdout.write(
" * Reassigning {} files to the user library...".format(total)
)
user_files.update(library=library)
files = models.TrackFile.objects.filter(
library__isnull=True, jobs__isnull=True
).distinct()
command.stdout.write(
"* Handling {} files with no import jobs...".format(files.count())
)
user = User.objects.order_by("id").filter(is_superuser=True).first()
command.stdout.write(" * Setting up @{}'s 'default' library".format(user.username))
library = user.actor.libraries.get_or_create(actor=user.actor, name="default")[0]
total = files.count()
command.stdout.write(
" * Reassigning {} files to the user library...".format(total)
)
files.update(library=library)
command.stdout.write(" * Done!")
......@@ -54,6 +54,7 @@ class ActionSerializer(serializers.Serializer):
objects = serializers.JSONField(required=True)
filters = serializers.DictField(required=False)
actions = None
pk_field = "pk"
def __init__(self, *args, **kwargs):
self.actions_by_name = {a.name: a for a in self.actions}
......@@ -84,7 +85,9 @@ class ActionSerializer(serializers.Serializer):
if value == "all":
return self.queryset.all().order_by("id")
if type(value) in [list, tuple]:
return self.queryset.filter(pk__in=value).order_by("id")
return self.queryset.filter(
**{"{}__in".format(self.pk_field): value}
).order_by("id")
raise serializers.ValidationError(
"{} is not a valid value for objects. You must provide either a "
......
......@@ -3,9 +3,12 @@ from rest_framework.decorators import list_route
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from rest_framework.response import Response
from django.db.models import Prefetch
from funkwhale_api.activity import record
from funkwhale_api.common import fields, permissions
from funkwhale_api.music.models import Track
from funkwhale_api.music import utils as music_utils
from . import filters, models, serializers
......@@ -19,11 +22,7 @@ class TrackFavoriteViewSet(
filter_class = filters.TrackFavoriteFilter
serializer_class = serializers.UserTrackFavoriteSerializer
queryset = (
models.TrackFavorite.objects.all()
.select_related("track__artist", "track__album__artist", "user")
.prefetch_related("track__files")
)
queryset = models.TrackFavorite.objects.all().select_related("user")
permission_classes = [
permissions.ConditionalAuthentication,
permissions.OwnerPermission,
......@@ -49,9 +48,14 @@ class TrackFavoriteViewSet(
def get_queryset(self):
queryset = super().get_queryset()
return queryset.filter(
queryset = queryset.filter(
fields.privacy_level_query(self.request.user, "user__privacy_level")
)
tracks = Track.objects.annotate_playable_by_actor(
music_utils.get_actor_from_request(self.request)
).select_related("artist", "album__artist")
queryset = queryset.prefetch_related(Prefetch("track", queryset=tracks))
return queryset
def perform_create(self, serializer):
track = Track.objects.get(pk=serializer.data["track"])
......
from funkwhale_api.common import utils as funkwhale_utils
ACTIVITY_TYPES = [
"Accept",
"Add",
......@@ -61,21 +63,21 @@ def accept_follow(follow):
return deliver(serializer.data, to=[follow.actor.fid], on_behalf_of=follow.target)
def receive(activity, on_behalf_of):
def receive(activity, on_behalf_of, recipient):
from . import serializers
from . import tasks
# we ensure the activity has the bare minimum structure before storing
# it in our database
serializer = serializers.BaseActivitySerializer(
data=activity, context={"actor": on_behalf_of}
data=activity, context={"actor": on_behalf_of, "recipient": recipient}
)
serializer.is_valid(raise_exception=True)
copy = serializer.save()
# at this point, we have the activity in database. Even if we crash, it's
# okay, as we can retry later
tasks.dispatch.delay(activity_id=copy.pk)
tasks.dispatch_inbox.delay(activity_id=copy.pk)
return copy
......@@ -86,12 +88,51 @@ class Router:
def connect(self, route, handler):
self.routes.append((route, handler))
def register(self, route):
def decorator(handler):
self.connect(route, handler)
return handler
return decorator
class InboxRouter(Router):
def dispatch(self, payload, context):
"""
Receives an Activity payload and some context and trigger our
business logic
"""
for route, handler in self.routes:
if match_route(route, payload):
return handler(payload, context=context)
class OutboxRouter(Router):
def dispatch(self, routing, context):
"""
Receives a routing payload and some business objects in the context
and may yield data that should be persisted in the Activity model
for further delivery.
"""
from . import models
from . import tasks
for route, handler in self.routes:
if match_route(route, routing):
activities_data = []
for e in handler(context):
activities_data.append(models.Activity(**e))
activities = models.Activity.objects.bulk_create(activities_data)
for a in activities:
if not a:
continue
funkwhale_utils.on_commit(
tasks.dispatch_outbox.delay, activity_id=a.pk
)
return activities
def match_route(route, payload):
for key, value in route.items():
if payload.get(key) != value:
......
......@@ -8,6 +8,7 @@ from django.utils import timezone
from django.utils.http import http_date
from funkwhale_api.factories import registry
from funkwhale_api.users import factories as user_factories
from . import keys, models
......@@ -61,6 +62,10 @@ class LinkFactory(factory.Factory):
audio = factory.Trait(mediaType=factory.Iterator(["audio/mp3", "audio/ogg"]))
def create_user(actor):
return user_factories.UserFactory(actor=actor)
@registry.register
class ActorFactory(factory.DjangoModelFactory):
public_key = None
......@@ -115,6 +120,7 @@ class MusicLibraryFactory(factory.django.DjangoModelFactory):
privacy_level = "me"
name = factory.Faker("sentence")
description = factory.Faker("sentence")
files_count = 0
class Meta:
model = "music.Library"
......@@ -136,6 +142,15 @@ class MusicLibraryFactory(factory.django.DjangoModelFactory):
self.followers_url = extracted or self.fid + "/followers"
@registry.register
class LibraryScan(factory.django.DjangoModelFactory):
library = factory.SubFactory(MusicLibraryFactory)
total_files = factory.LazyAttribute(lambda o: o.library.files_count)
class Meta:
model = "music.LibraryScan"
@registry.register
class ActivityFactory(factory.django.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
......@@ -152,7 +167,7 @@ class LibraryFollowFactory(factory.DjangoModelFactory):
actor = factory.SubFactory(ActorFactory)
class Meta:
model = models.LibraryFollow
model = "federation.LibraryFollow"
@registry.register
......
......@@ -7,23 +7,41 @@ import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('music', '0029_auto_20180807_1748'),
('federation', '0007_auto_20180807_1748'),
("music", "0029_auto_20180807_1748"),
("federation", "0007_auto_20180807_1748"),
]
operations = [
migrations.AddField(
model_name='libraryfollow',
name='target',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='received_follows', to='music.Library'),
model_name="libraryfollow",
name="target",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="received_follows",
to="music.Library",
),
),
migrations.AddField(
model_name='activity',
name='actor',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activities', to='federation.Actor'),
model_name="activity",
name="actor",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="activities",
to="federation.Actor",
),
),
migrations.AddField(
model_name="activity",
name="recipient",
field=models.ForeignKey(
on_delete=django.db.models.deletion.SET_NULL,
related_name="inbox_activities",
to="federation.Actor",
null=True,
blank=True,
),
),
migrations.AlterUniqueTogether(
name='libraryfollow',
unique_together={('actor', 'target')},
name="libraryfollow", unique_together={("actor", "target")}
),
]
......@@ -11,6 +11,8 @@ from funkwhale_api.common import session
from funkwhale_api.common import utils as common_utils
from funkwhale_api.music import utils as music_utils
from . import utils as federation_utils
TYPE_CHOICES = [
("Person", "Person"),
("Application", "Application"),
......@@ -117,11 +119,22 @@ class Actor(models.Model):
follows = self.received_follows.filter(approved=True)
return self.followers.filter(pk__in=follows.values_list("actor", flat=True))
def should_autoapprove_follow(self, actor):
return False
class Activity(models.Model):
actor = models.ForeignKey(
Actor, related_name="activities", on_delete=models.CASCADE
)
# the actor who own the inbox the activity was published to
recipient = models.ForeignKey(
Actor,
related_name="inbox_activities",
on_delete=models.SET_NULL,
null=True,
blank=True,
)
uuid = models.UUIDField(default=uuid.uuid4, unique=True)
fid = models.URLField(unique=True, max_length=500, null=True, blank=True)
url = models.URLField(max_length=500, null=True, blank=True)
......@@ -143,7 +156,9 @@ class AbstractFollow(models.Model):
abstract = True
def get_federation_id(self):
return "{}#follows/{}".format(self.actor.fid, self.uuid)
return federation_utils.full_url(
"{}#follows/{}".format(self.actor.fid, self.uuid)
)
class Follow(AbstractFollow):
......
import logging
from . import activity
from . import serializers
logger = logging.getLogger(__name__)
inbox = activity.InboxRouter()
outbox = activity.OutboxRouter()
@inbox.register({"type": "Follow"})
def inbox_follow(payload, context):
serializer = serializers.FollowSerializer(data=payload, context=context)
if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
logger.debug(
"Discarding invalid follow from {}: %s",
context["actor"].fid,
serializer.errors,
)
return
autoapprove = serializer.validated_data["object"].should_autoapprove_follow(
context["actor"]
)
follow = serializer.save(approved=autoapprove)
if autoapprove:
activity.accept_follow(follow)
@inbox.register({"type": "Accept"})
def inbox_accept(payload, context):
serializer = serializers.AcceptFollowSerializer(data=payload, context=context)
if not serializer.is_valid(raise_exception=context.get("raise_exception", False)):
logger.debug(
"Discarding invalid accept from {}: %s",
context["actor"].fid,
serializer.errors,
)
return
serializer.save()
@outbox.register({"type": "Accept"})
def outbox_accept(context):
follow = context["follow"]
if follow._meta.label == "federation.LibraryFollow":
actor = follow.target.actor
else:
actor = follow.target
router = activity.Router()
yield {
"actor": actor,
"recipient": follow.actor,
"payload": serializers.AcceptFollowSerializer(
follow, context={"actor": actor}
).data,
}
......@@ -156,6 +156,7 @@ class BaseActivitySerializer(serializers.Serializer):
return models.Activity.objects.create(
fid=validated_data.get("id"),
actor=validated_data["actor"],
recipient=self.context.get("recipient"),
payload=self.initial_data,
)
......@@ -351,12 +352,30 @@ class FollowSerializer(serializers.Serializer):
def validate_object(self, v):
expected = self.context.get("follow_target")
if self.parent:
# it's probably an accept, so everything is inverted, the actor
# the recipient does not matter
recipient = None
else:
recipient = self.context.get("recipient")
if expected and expected.fid != v:
raise serializers.ValidationError("Invalid target")
try:
return models.Actor.objects.get(fid=v)
obj = models.Actor.objects.get(fid=v)
if recipient and recipient.fid != obj.fid:
raise serializers.ValidationError("Invalid target")
return obj
except models.Actor.DoesNotExist:
raise serializers.ValidationError("Target not found")
pass
try:
qs = music_models.Library.objects.filter(fid=v)
if recipient:
qs = qs.filter(actor=recipient)
return qs.get()
except music_models.Library.DoesNotExist:
pass
raise serializers.ValidationError("Target not found")
def validate_actor(self, v):
expected = self.context.get("follow_actor")
......@@ -368,10 +387,18 @@ class FollowSerializer(serializers.Serializer):
raise serializers.ValidationError("Actor not found")
def save(self, **kwargs):
return models.Follow.objects.get_or_create(
target = self.validated_data["object"]
if target._meta.label == "music.Library":
follow_class = models.LibraryFollow
else:
follow_class = models.Follow
defaults = kwargs
defaults["fid"] = self.validated_data["id"]
return follow_class.objects.update_or_create(
actor=self.validated_data["actor"],
target=self.validated_data["object"],
**kwargs, # noqa
defaults=defaults,
)[0]
def to_representation(self, instance):
......@@ -402,13 +429,13 @@ class APIFollowSerializer(serializers.ModelSerializer):
class AcceptFollowSerializer(serializers.Serializer):
id = serializers.URLField(max_length=500)
id = serializers.URLField(max_length=500, required=False)
actor = serializers.URLField(max_length=500)
object = FollowSerializer()
type = serializers.ChoiceField(choices=["Accept"])
def validate_actor(self, v):
expected = self.context.get("follow_target")
expected = self.context.get("actor")
if expected and expected.fid != v:
raise serializers.ValidationError("Invalid actor")
try:
......@@ -417,19 +444,27 @@ class AcceptFollowSerializer(serializers.Serializer):
raise serializers.ValidationError("Actor not found")
def validate(self, validated_data):
# we ensure the accept actor actually match the follow target
if validated_data["actor"] != validated_data["object"]["object"]:
# we ensure the accept actor actually match the follow target / library owner
target = validated_data["object"]["object"]
if target._meta.label == "music.Library":
expected = target.actor
follow_class = models.LibraryFollow
else:
expected = target
follow_class = models.Follow
if validated_data["actor"] != expected:
raise serializers.ValidationError("Actor mismatch")
try:
validated_data["follow"] = (
models.Follow.objects.filter(
target=validated_data["actor"],
actor=validated_data["object"]["actor"],
follow_class.objects.filter(
target=target, actor=validated_data["object"]["actor"]
)
.exclude(approved=True)
.select_related()
.get()
)
except models.Follow.DoesNotExist:
except follow_class.DoesNotExist:
raise serializers.ValidationError("No follow to accept")
return validated_data
......@@ -438,14 +473,17 @@ class AcceptFollowSerializer(serializers.Serializer):
"@context": AP_CONTEXT,
"id": instance.get_federation_id() + "/accept",
"type": "Accept",
"actor": instance.target.fid,
"actor": self.context["actor"].fid,
"object": FollowSerializer(instance).data,
}
def save(self):
self.validated_data["follow"].approved = True
self.validated_data["follow"].save()
return self.validated_data["follow"]
follow = self.validated_data["follow"]
follow.approved = True
follow.save()
if follow.target._meta.label == "music.Library":
follow.target.schedule_scan()
return follow
class UndoFollowSerializer(serializers.Serializer):
......
......@@ -133,18 +133,21 @@ def get_files(storage, *parts):
return [os.path.join(parts[-1], path) for path in files]
@celery.app.task(name="federation.dispatch")
@celery.app.task(name="federation.dispatch_inbox")
@celery.require_instance(
models.Activity.objects.exclude(delivered=True).select_related("actor"), "activity"
models.Activity.objects.exclude(delivered=True).select_related(), "activity"
)
def dispatch(activity):
def dispatch_inbox(activity):
"""
Given an activity instance, triggers our internal delivery logic (follow
creation, etc.)
"""
try:
routes.router.dispatch(activity.payload, context={"actor": activity.actor})
routes.inbox.dispatch(
activity.payload,
context={"actor": activity.actor, "recipient": activity.recipient},
)
except Exception:
activity.delivered = False
activity.save(update_fields=["delivered"])
......@@ -153,3 +156,14 @@ def dispatch(activity):
activity.delivered = True
activity.delivered_date = timezone.now()
activity.save(update_fields=["delivered", "delivered_date"])
@celery.app.task(name="federation.dispatch_outbox")
@celery.require_instance(
models.Activity.objects.exclude(delivered=True).select_related(), "activity"
)
def dispatch_outbox(activity):
"""
Deliver a local activity to its recipients
"""
pass
......@@ -12,6 +12,7 @@ router.register(r"federation/actors", views.ActorViewSet, "actors")
router.register(r".well-known", views.WellKnownViewSet, "well-known")
music_router.register(r"files", views.MusicFilesViewSet, "files")
music_router.register(r"libraries", views.MusicLibraryViewSet, "libraries")
urlpatterns = router.urls + [
url("federation/music/", include((music_router.urls, "music"), namespace="music"))
]
......@@ -48,12 +48,15 @@ class ActorViewSet(
@detail_route(methods=["get", "post"])
def inbox(self, request, *args, **kwargs):
actor = self.get_object()
if request.method.lower() == "post" and request.actor is None:
raise exceptions.AuthenticationFailed(
"You need a valid signature to send an activity"
)
if request.method.lower() == "post":
activity.receive(activity=request.data, on_behalf_of=request.actor)
activity.receive(
activity=request.data, on_behalf_of=request.actor, recipient=actor
)
return response.Response({}, status=200)
@detail_route(methods=["get", "post"])
......@@ -201,12 +204,79 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet):
return response.Response(data)
def has_library_access(request, library):
if library.privacy_level == "everyone":
return True
if request.user.is_authenticated and request.user.is_superuser:
return True
try:
actor = request.actor
except AttributeError:
return False
return library.received_follows.filter(actor=actor, approved=True).exists()
class MusicLibraryViewSet(
FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
):
authentication_classes = [authentication.SignatureAuthentication]
permission_classes = []
renderer_classes = [renderers.ActivityPubRenderer]
serializer_class = serializers.PaginatedCollectionSerializer
queryset = music_models.Library.objects.all().select_related("actor")
lookup_field = "uuid"
def retrieve(self, request, *args, **kwargs):
library = self.get_object()
conf = {
"id": library.get_federation_id(),
"actor": library.actor,
"name": library.name,
"summary": library.description,
"items": library.files.order_by("-creation_date"),
"item_serializer": serializers.AudioSerializer,
}
page = request.GET.get("page")
if page is None:
serializer = serializers.PaginatedCollectionSerializer(conf)
data = serializer.data
else:
# if actor is requesting a specific page, we ensure library is public
# or readable by the actor
if not has_library_access(request, library):
raise exceptions.AuthenticationFailed(
"You do not have access to this library"
)
try:
page_number = int(page)
except Exception:
return response.Response({"page": ["Invalid page number"]}, status=400)
conf["page_size"] = preferences.get("federation__collection_page_size")
p = paginator.Paginator(conf["items"], conf["page_size"])
try:
page = p.page(page_number)
conf["page"] = page
serializer = serializers.CollectionPageSerializer(conf)
data = serializer.data
except paginator.EmptyPage:
return response.Response(status=404)
return response.Response(data)
class LibraryViewSet(
mixins.RetrieveModelMixin,
mixins.UpdateModelMixin,
mixins.ListModelMixin,
viewsets.GenericViewSet,
):
"""