I've managed to figure this out, so I'll post my code here for anyone who can use it:

So I just created a User Control, frontend:

<%@ Control Language="C#" AutoEventWireup="true" CodeBehind="SilverlightControl.ascx.cs" 
Inherits="V.Templates.DynamicContent.SilverlightControl" %>

<div id="silverlightControlHost" runat="server">
    <object data="data:application/x-silverlight-2," type="application/x-silverlight-2" width="550px" height="400px">
    <param name="source" value="/control.xap"/>
    <param name="onError" value="onSilverlightError" />
    <param name="background" value="#eeeeee" />
    <param name="minRuntimeVersion" value="4.0.50826.0" />
    <param name="autoUpgrade" value="true" />
    <a href="http://go.microsoft.com/fwlink/?LinkID=149156&v=4.0.50826.0" style="text-decoration:none">
        <img src="http://go.microsoft.com/fwlink/?LinkId=161376" alt="Get Microsoft Silverlight" style="border-style:none"/>
    </a>
    </object>

    <iframe id="_sl_historyFrame" style="visibility:hidden;height:0px;width:0px;border:0px"></iframe>
</div>

and backend:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using EPiServer.DynamicContent;
using EPiServer.PlugIn;

namespace V.Templates.DynamicContent
{
    [DynamicContentPlugIn(
            DisplayName = "Silverlight component",
            Description = "Silverlight component",
            ViewUrl = "~/Templates/DynamicContent/SilverlightControl.ascx")]

    public partial class SilverlightControl : EPiServer.UserControlBase
    {
        protected override void OnLoad(EventArgs e)
        {
            base.OnLoad(e);
        }
    }
}

The position of the .xap-file is really important, so make sure you put this in the right place.