diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 69365017..f5500157 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -24,7 +24,6 @@ If applicable, add screenshots to help explain your problem. **Environment (please complete the following information):** - OS: [e.g. Ubuntu Linux] - - Installation method: [Docker install, or single server install] - Browser, if applicable **Additional context** diff --git a/README.md b/README.md index 962e4262..52d6aec5 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ A demo is available at https://demo.mediacms.io - **Multiple transcoding profiles**: sane defaults for multiple dimensions (144p, 240p, 360p, 480p, 720p, 1080p) and multiple profiles (h264, h265, vp9) - **Adaptive video streaming**: possible through HLS protocol - **Subtitles/CC**: support for multilingual subtitle files -- **Scalable transcoding**: transcoding through priorities. Experimental support for remote workers +- **Scalable transcoding**: transcoding through priorities. - **Chunked file uploads**: for pausable/resumable upload of content - **REST API**: Documented through Swagger - **Translation**: Most of the CMS is translated to a number of languages @@ -91,7 +91,6 @@ In order to support automatic transcriptions through Whisper, consider more CPUs There are two ways to run MediaCMS, through Docker Compose and through installing it on a server via an automation script that installs and configures all needed services. Find the related pages: -- [Single Server](docs/admins_docs.md#2-server-installation) page - [Docker Compose](docs/admins_docs.md#3-docker-installation) page A complete guide can be found on the blog post [How to self-host and share your videos in 2021](https://medium.com/@MediaCMS.io/how-to-self-host-and-share-your-videos-in-2021-14067e3b291b). diff --git a/cms/settings.py b/cms/settings.py index bda55ef0..480ed02d 100644 --- a/cms/settings.py +++ b/cms/settings.py @@ -266,21 +266,6 @@ CANNOT_ADD_MEDIA_MESSAGE = "User cannot add media, or maximum number of media up # mp4hls command, part of Bento4 MP4HLS_COMMAND = "/home/mediacms.io/mediacms/Bento4-SDK-1-6-0-637.x86_64-unknown-linux/bin/mp4hls" -# highly experimental, related with remote workers -ADMIN_TOKEN = "" -# this is used by remote workers to push -# encodings once they are done -# USE_BASIC_HTTP = True -# BASIC_HTTP_USER_PAIR = ('user', 'password') -# specify basic auth user/password pair for use with the -# remote workers, if nginx basic auth is setup -# apache2-utils need be installed -# then run -# htpasswd -c /home/mediacms.io/mediacms/deploy/.htpasswd user -# and set a password -# edit /etc/nginx/sites-enabled/mediacms.io and -# uncomment the two lines related to htpasswd - AUTH_USER_MODEL = "users.User" LOGIN_REDIRECT_URL = "/" diff --git a/docs/admins_docs.md b/docs/admins_docs.md index 6ae1bd92..8709c60a 100644 --- a/docs/admins_docs.md +++ b/docs/admins_docs.md @@ -2,7 +2,6 @@ ## Table of contents - [1. Welcome](#1-welcome) -- [2. Single Server Installaton](#2-single-server-installation) - [3. Docker Installation](#3-docker-installation) - [4. Docker Deployment options](#4-docker-deployment-options) - [5. Configuration](#5-configuration) @@ -34,58 +33,6 @@ ## 1. Welcome This page is created for MediaCMS administrators that are responsible for setting up the software, maintaining it and making modifications. -## 2. Single Server Installation - -The core dependencies are python3, Django, celery, PostgreSQL, redis, ffmpeg. Any system that can have these dependencies installed, can run MediaCMS. But the install.sh is only tested in Linux Ubuntu 24 and 22 versions. - -Installation on an Ubuntu 22/24 system with git utility installed should be completed in a few minutes with the following steps. -Make sure you run it as user root, on a clear system, since the automatic script will install and configure the following services: Celery/PostgreSQL/Redis/Nginx and will override any existing settings. - - - -```bash -mkdir /home/mediacms.io && cd /home/mediacms.io/ -git clone https://github.com/mediacms-io/mediacms -cd /home/mediacms.io/mediacms/ && bash ./install.sh -``` - -The script will ask if you have a URL where you want to deploy MediaCMS, otherwise it will use localhost. If you provide a URL, it will use Let's Encrypt service to install a valid ssl certificate. - - -### Update - -If you've used the above way to install MediaCMS, update with the following: - -```bash -cd /home/mediacms.io/mediacms # enter mediacms directory -source /home/mediacms.io/bin/activate # use virtualenv -git pull # update code -pip install -r requirements.txt -U # run pip install to update -python manage.py migrate # run Django migrations -sudo systemctl restart mediacms celery_long celery_short # restart services -``` - -### Update from version 2 to version 3 -Version 3 is using Django 4 and Celery 5, and needs a recent Python 3.x version. If you are updating from an older version, make sure Python is updated first. Version 2 could run on Python 3.6, but version 3 needs Python3.8 and higher. -The syntax for starting Celery has also changed, so you have to copy the celery related systemctl files and restart - -``` -# cp deploy/local_install/celery_long.service /etc/systemd/system/celery_long.service -# cp deploy/local_install/celery_short.service /etc/systemd/system/celery_short.service -# cp deploy/local_install/celery_beat.service /etc/systemd/system/celery_beat.service -# systemctl daemon-reload -# systemctl start celery_long celery_short celery_beat -``` - - - -### Configuration -Checkout the configuration section here. - - -### Maintenance -Database can be backed up with pg_dump and media_files on /home/mediacms.io/mediacms/media_files include original files and encoded/transcoded versions - ## 3. Docker Installation @@ -220,14 +167,10 @@ Several options are available on `cms/settings.py`, most of the things that are It is advisable to override any of them by adding it to `local_settings.py` . -In case of a the single server installation, add to `cms/local_settings.py` . - In case of a docker compose installation, add to `deploy/docker/local_settings.py` . This will automatically overwrite `cms/local_settings.py` . Any change needs restart of MediaCMS in order to take effect. -Single server installation: edit `cms/local_settings.py`, make a change and restart MediaCMS - ```bash #systemctl restart mediacms ``` @@ -795,14 +738,7 @@ Instructions contributed by @alberto98fx On the [Configuration](https://github.com/mediacms-io/mediacms/blob/main/docs/admins_docs.md#5-configuration) section of this guide we've see how to edit the email settings. In case we are yet unable to receive email from MediaCMS, the following may help us debug the issue - in most cases it is an issue of setting the correct username, password or TLS option -Enter the Django shell, example if you're using the Single Server installation: - -```bash -source /home/mediacms.io/bin/activate -python manage.py shell -``` - -and inside the shell +Enter the Django shell and inside the shell ```bash from django.core.mail import EmailMessage diff --git a/files/management_views.py b/files/management_views.py index 8339ba1f..fc3a3131 100644 --- a/files/management_views.py +++ b/files/management_views.py @@ -120,9 +120,11 @@ class MediaList(APIView): operation_description='Delete media for MediaCMS managers and reviewers', ) def delete(self, request, format=None): + if not is_mediacms_manager(request.user): + return Response({"detail": "bad permissions"}, status=status.HTTP_403_FORBIDDEN) tokens = request.GET.get("tokens") if tokens: - tokens = tokens.split(",") + tokens = [t for t in tokens.split(",") if t][:50] Media.objects.filter(friendly_token__in=tokens).delete() return Response(status=status.HTTP_204_NO_CONTENT) @@ -177,7 +179,7 @@ class CommentList(APIView): def delete(self, request, format=None): comment_ids = request.GET.get("comment_ids") if comment_ids: - comments = comment_ids.split(",") + comments = [c for c in comment_ids.split(",") if c][:50] Comment.objects.filter(uid__in=comments).delete() return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/files/methods.py b/files/methods.py index 4c2783a0..5a7bffbb 100644 --- a/files/methods.py +++ b/files/methods.py @@ -453,17 +453,21 @@ def kill_ffmpeg_process(filepath): filepath: Path to the file being processed by ffmpeg Returns: - subprocess.CompletedProcess: Result of the kill command + bool: True if the lookup ran, False if input was unusable """ - if not filepath: + if not filepath or not isinstance(filepath, str): return False - cmd = "ps aux|grep 'ffmpeg'|grep %s|grep -v grep |awk '{print $2}'" % filepath - result = subprocess.run(cmd, stdout=subprocess.PIPE, shell=True) - pid = result.stdout.decode("utf-8").strip() - if pid: - cmd = "kill -9 %s" % pid - result = subprocess.run(cmd, stdout=subprocess.PIPE, shell=True) - return result + try: + ps = subprocess.run(["ps", "aux"], stdout=subprocess.PIPE, check=False) + except OSError: + return False + for line in ps.stdout.decode("utf-8", "replace").splitlines(): + if "ffmpeg" not in line or filepath not in line or "grep" in line: + continue + parts = line.split() + if len(parts) > 1 and parts[1].isdigit(): + subprocess.run(["kill", "-9", parts[1]], check=False) + return True def copy_video(original_media, copy_encodings=True, title_suffix="(Trimmed)"): diff --git a/files/models/encoding.py b/files/models/encoding.py index ec209659..e444168d 100644 --- a/files/models/encoding.py +++ b/files/models/encoding.py @@ -6,7 +6,6 @@ from django.core.files import File from django.db import models from django.db.models.signals import post_delete, post_save from django.dispatch import receiver -from django.urls import reverse from .. import helpers from .utils import ( @@ -136,9 +135,6 @@ class Encoding(models.Model): def __str__(self): return f"{self.profile.name}-{self.media.title}" - def get_absolute_url(self): - return reverse("api_get_encoding", kwargs={"encoding_id": self.id}) - @receiver(post_save, sender=Encoding) def encoding_file_save(sender, instance, created, **kwargs): diff --git a/files/models/media.py b/files/models/media.py index 7f1e0a20..4d505657 100644 --- a/files/models/media.py +++ b/files/models/media.py @@ -559,9 +559,8 @@ class Media(models.Model): profiles.remove(profile) encoding = Encoding(media=self, profile=profile) encoding.save() - enc_url = settings.SSL_FRONTEND_HOST + encoding.get_absolute_url() tasks.encode_media.apply_async( - args=[self.friendly_token, profile.id, encoding.id, enc_url], + args=[self.friendly_token, profile.id, encoding.id], kwargs={"force": force}, priority=0, ) @@ -575,13 +574,12 @@ class Media(models.Model): continue encoding = Encoding(media=self, profile=profile) encoding.save() - enc_url = settings.SSL_FRONTEND_HOST + encoding.get_absolute_url() if profile.resolution in settings.MINIMUM_RESOLUTIONS_TO_ENCODE: priority = 9 else: priority = 0 tasks.encode_media.apply_async( - args=[self.friendly_token, profile.id, encoding.id, enc_url], + args=[self.friendly_token, profile.id, encoding.id], kwargs={"force": force}, priority=priority, ) diff --git a/files/tasks.py b/files/tasks.py index 2a0c9398..503ced02 100644 --- a/files/tasks.py +++ b/files/tasks.py @@ -171,8 +171,7 @@ def chunkize_media(self, friendly_token, profiles, force=True): continue encoding = Encoding(media=media, profile=profile) encoding.save() - enc_url = settings.SSL_FRONTEND_HOST + encoding.get_absolute_url() - encode_media.delay(friendly_token, profile.id, encoding.id, enc_url, force=force) + encode_media.delay(friendly_token, profile.id, encoding.id, force=force) return False chunks = [os.path.join(cwd, ch) for ch in chunks] @@ -202,13 +201,12 @@ def chunkize_media(self, friendly_token, profiles, force=True): ) encoding.save() - enc_url = settings.SSL_FRONTEND_HOST + encoding.get_absolute_url() if profile.resolution in settings.MINIMUM_RESOLUTIONS_TO_ENCODE: priority = 0 else: priority = 9 encode_media.apply_async( - args=[friendly_token, profile.id, encoding.id, enc_url], + args=[friendly_token, profile.id, encoding.id], kwargs={"force": force, "chunk": True, "chunk_file_path": chunk}, priority=priority, ) @@ -246,7 +244,6 @@ def encode_media( friendly_token, profile_id, encoding_id, - encoding_url, force=True, chunk=False, chunk_file_path="", diff --git a/files/urls.py b/files/urls.py index 4ad50a62..e61b9dcf 100644 --- a/files/urls.py +++ b/files/urls.py @@ -61,11 +61,6 @@ urlpatterns = [ views.MediaDetail.as_view(), name="api_get_media", ), - re_path( - r"^api/v1/media/encoding/(?P[\w]*)$", - views.EncodingDetail.as_view(), - name="api_get_encoding", - ), re_path(r"^api/v1/search$", views.MediaSearch.as_view()), re_path( rf"^api/v1/media/{friendly_token}/share$", diff --git a/files/views/__init__.py b/files/views/__init__.py index fb1d996d..db5651ba 100644 --- a/files/views/__init__.py +++ b/files/views/__init__.py @@ -3,7 +3,7 @@ from .auth import custom_login_view, saml_metadata # noqa: F401 from .categories import CategoryList, CategoryListContributor, TagList # noqa: F401 from .comments import CommentDetail, CommentList # noqa: F401 -from .encoding import EncodeProfileList, EncodingDetail # noqa: F401 +from .encoding import EncodeProfileList # noqa: F401 from .media import MediaActions # noqa: F401 from .media import MediaBulkUserActions # noqa: F401 from .media import MediaDetail # noqa: F401 diff --git a/files/views/encoding.py b/files/views/encoding.py index abea3fb3..fcb9e05b 100644 --- a/files/views/encoding.py +++ b/files/views/encoding.py @@ -1,168 +1,11 @@ -from django.conf import settings from drf_yasg.utils import swagger_auto_schema -from rest_framework import permissions, status -from rest_framework.parsers import ( - FileUploadParser, - FormParser, - JSONParser, - MultiPartParser, -) from rest_framework.response import Response from rest_framework.views import APIView -from ..helpers import produce_ffmpeg_commands -from ..models import EncodeProfile, Encoding +from ..models import EncodeProfile from ..serializers import EncodeProfileSerializer -class EncodingDetail(APIView): - """Experimental. This View is used by remote workers - Needs heavy testing and documentation. - """ - - permission_classes = (permissions.IsAdminUser,) - parser_classes = (JSONParser, MultiPartParser, FormParser, FileUploadParser) - - @swagger_auto_schema(auto_schema=None) - def post(self, request, encoding_id): - ret = {} - force = request.data.get("force", False) - task_id = request.data.get("task_id", False) - action = request.data.get("action", "") - chunk = request.data.get("chunk", False) - chunk_file_path = request.data.get("chunk_file_path", "") - - encoding_status = request.data.get("status", "") - progress = request.data.get("progress", "") - commands = request.data.get("commands", "") - logs = request.data.get("logs", "") - retries = request.data.get("retries", "") - worker = request.data.get("worker", "") - temp_file = request.data.get("temp_file", "") - total_run_time = request.data.get("total_run_time", "") - if action == "start": - try: - encoding = Encoding.objects.get(id=encoding_id) - media = encoding.media - profile = encoding.profile - except BaseException: - Encoding.objects.filter(id=encoding_id).delete() - return Response({"status": "fail"}, status=status.HTTP_400_BAD_REQUEST) - # TODO: break chunk True/False logic here - if ( - Encoding.objects.filter( - media=media, - profile=profile, - chunk=chunk, - chunk_file_path=chunk_file_path, - ).count() - > 1 # noqa - and force is False # noqa - ): - Encoding.objects.filter(id=encoding_id).delete() - return Response({"status": "fail"}, status=status.HTTP_400_BAD_REQUEST) - else: - Encoding.objects.filter( - media=media, - profile=profile, - chunk=chunk, - chunk_file_path=chunk_file_path, - ).exclude(id=encoding.id).delete() - - encoding.status = "running" - if task_id: - encoding.task_id = task_id - - encoding.save() - if chunk: - original_media_path = chunk_file_path - original_media_md5sum = encoding.md5sum - original_media_url = settings.SSL_FRONTEND_HOST + encoding.media_chunk_url - else: - original_media_path = media.media_file.path - original_media_md5sum = media.md5sum - original_media_url = settings.SSL_FRONTEND_HOST + media.original_media_url - - ret["original_media_url"] = original_media_url - ret["original_media_path"] = original_media_path - ret["original_media_md5sum"] = original_media_md5sum - - # generating the commands here, and will replace these with temporary - # files created on the remote server - tf = "TEMP_FILE_REPLACE" - tfpass = "TEMP_FPASS_FILE_REPLACE" - ffmpeg_commands = produce_ffmpeg_commands( - original_media_path, - media.media_info, - resolution=profile.resolution, - codec=profile.codec, - output_filename=tf, - pass_file=tfpass, - chunk=chunk, - ) - if not ffmpeg_commands: - encoding.delete() - return Response({"status": "fail"}, status=status.HTTP_400_BAD_REQUEST) - - ret["duration"] = media.duration - ret["ffmpeg_commands"] = ffmpeg_commands - ret["profile_extension"] = profile.extension - return Response(ret, status=status.HTTP_201_CREATED) - elif action == "update_fields": - try: - encoding = Encoding.objects.get(id=encoding_id) - except BaseException: - return Response({"status": "fail"}, status=status.HTTP_400_BAD_REQUEST) - to_update = ["size", "update_date"] - if encoding_status: - encoding.status = encoding_status - to_update.append("status") - if progress: - encoding.progress = progress - to_update.append("progress") - if logs: - encoding.logs = logs - to_update.append("logs") - if commands: - encoding.commands = commands - to_update.append("commands") - if task_id: - encoding.task_id = task_id - to_update.append("task_id") - if total_run_time: - encoding.total_run_time = total_run_time - to_update.append("total_run_time") - if worker: - encoding.worker = worker - to_update.append("worker") - if temp_file: - encoding.temp_file = temp_file - to_update.append("temp_file") - - if retries: - encoding.retries = retries - to_update.append("retries") - - try: - encoding.save(update_fields=to_update) - except BaseException: - return Response({"status": "fail"}, status=status.HTTP_400_BAD_REQUEST) - return Response({"status": "success"}, status=status.HTTP_201_CREATED) - - @swagger_auto_schema(auto_schema=None) - def put(self, request, encoding_id, format=None): - encoding_file = request.data["file"] - encoding = Encoding.objects.filter(id=encoding_id).first() - if not encoding: - return Response( - {"detail": "encoding does not exist"}, - status=status.HTTP_400_BAD_REQUEST, - ) - encoding.media_file = encoding_file - encoding.save() - return Response({"detail": "ok"}, status=status.HTTP_201_CREATED) - - class EncodeProfileList(APIView): """List encode profiles""" diff --git a/frontend/src/static/js/utils/helpers/translate.js b/frontend/src/static/js/utils/helpers/translate.js index 6aca7098..e1659167 100644 --- a/frontend/src/static/js/utils/helpers/translate.js +++ b/frontend/src/static/js/utils/helpers/translate.js @@ -1,5 +1,5 @@ // check templates/config/installation/translations.html for more export function translateString(str) { - return window.TRANSLATION?.[str] || str; + return window.TRANSLATION?.[str] ?? str; } diff --git a/tests/test_imports.py b/tests/test_imports.py index 068bd8ee..e1db89ac 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -34,7 +34,6 @@ class TestImports(TestCase): from files.views import CommentDetail # noqa: F401 from files.views import CommentList # noqa: F401 from files.views import EncodeProfileList # noqa: F401 - from files.views import EncodingDetail # noqa: F401 from files.views import MediaActions # noqa: F401 from files.views import MediaBulkUserActions # noqa: F401 from files.views import MediaDetail # noqa: F401 diff --git a/users/views.py b/users/views.py index 419f24b1..1ecb1d6b 100644 --- a/users/views.py +++ b/users/views.py @@ -1,5 +1,7 @@ from django.conf import settings from django.contrib.auth.decorators import login_required +from django.contrib.auth.password_validation import validate_password +from django.core.exceptions import ValidationError as DjangoValidationError from django.core.mail import EmailMessage from django.db.models import Q from django.http import HttpResponseRedirect @@ -369,9 +371,15 @@ class UserDetail(APIView): if action == "change_password": # Permission to edit user is already checked by self.get_user -> self.check_object_permissions + if user.is_superuser and not request.user.is_superuser: + raise PermissionDenied("You do not have permission to change a superuser's password.") password = request.data.get("password") if not password: return Response({"detail": "Password is required"}, status=status.HTTP_400_BAD_REQUEST) + try: + validate_password(password, user=user) + except DjangoValidationError as exc: + return Response({"detail": list(exc.messages)}, status=status.HTTP_400_BAD_REQUEST) user.set_password(password) user.save()