diff options
| -rw-r--r-- | app/blueprints/admin/admin.py | 17 | ||||
| -rw-r--r-- | app/blueprints/packages/packages.py | 32 | ||||
| -rw-r--r-- | app/models.py | 11 | ||||
| -rw-r--r-- | app/tasks/importtasks.py | 55 | ||||
| -rw-r--r-- | app/templates/admin/list.html | 1 | ||||
| -rw-r--r-- | app/templates/packages/create_edit.html | 11 |
6 files changed, 123 insertions, 4 deletions
diff --git a/app/blueprints/admin/admin.py b/app/blueprints/admin/admin.py index e4fc78b..e0bc143 100644 --- a/app/blueprints/admin/admin.py +++ b/app/blueprints/admin/admin.py @@ -21,7 +21,7 @@ import flask_menu as menu from . import bp from app.models import * from celery import uuid, group -from app.tasks.importtasks import importRepoScreenshot, makeVCSRelease, checkZipRelease +from app.tasks.importtasks import importRepoScreenshot, makeVCSRelease, checkZipRelease, updateMetaFromRelease from app.tasks.forumtasks import importTopicList, checkAllForumAccounts from flask_wtf import FlaskForm from wtforms import * @@ -52,6 +52,21 @@ def admin_page(): time.sleep(0.1) return redirect(url_for("todo.view")) + elif action == "reimportpackages": + tasks = [] + for package in Package.query.filter_by(approved=True, soft_deleted=False).all(): + release = package.releases.first() + if release: + zippath = release.url.replace("/uploads/", app.config["UPLOAD_DIR"]) + tasks.append(updateMetaFromRelease.s(release.id, zippath)) + + result = group(tasks).apply_async() + + while not result.ready(): + import time + time.sleep(0.1) + + return redirect(url_for("todo.view")) elif action == "importmodlist": task = importTopicList.delay() return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics"))) diff --git a/app/blueprints/packages/packages.py b/app/blueprints/packages/packages.py index 78aa35c..464449b 100644 --- a/app/blueprints/packages/packages.py +++ b/app/blueprints/packages/packages.py @@ -23,7 +23,7 @@ from . import bp from app.models import * from app.querybuilder import QueryBuilder -from app.tasks.importtasks import importRepoScreenshot +from app.tasks.importtasks import importRepoScreenshot, updateMetaFromRelease from app.utils import * from flask_wtf import FlaskForm @@ -33,6 +33,8 @@ from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleF from sqlalchemy import or_, func from sqlalchemy.orm import joinedload, subqueryload +from celery import uuid + @menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' }) @menu.register_menu(bp, ".games", "Games", order=12, endpoint_arguments_constructor=lambda: { 'type': 'game' }) @@ -466,3 +468,31 @@ def remove_self_maintainers(package): db.session.commit() return redirect(package.getDetailsURL()) + + +@bp.route("/packages/<author>/<name>/import-meta/", methods=["POST"]) +@login_required +@is_package_page +def update_from_release(package): + if not package.checkPerm(current_user, Permission.REIMPORT_META): + flash("You don't have permission to reimport meta", "danger") + return redirect(package.getDetailsURL()) + + release = package.releases.first() + if not release: + flash("Release needed", "danger") + return redirect(package.getDetailsURL()) + + msg = "Updated meta from latest release" + addNotification(package.maintainers, current_user, + msg, package.getDetailsURL(), package) + severity = AuditSeverity.NORMAL if current_user in package.maintainers else AuditSeverity.EDITOR + addAuditLog(severity, current_user, msg, package.getDetailsURL(), package) + + db.session.commit() + + task_id = uuid() + zippath = release.url.replace("/uploads/", app.config["UPLOAD_DIR"]) + updateMetaFromRelease.apply_async((release.id, zippath), task_id=task_id) + + return redirect(url_for("tasks.check", id=task_id, r=package.getEditURL())) diff --git a/app/models.py b/app/models.py index d508619..4102049 100644 --- a/app/models.py +++ b/app/models.py @@ -80,6 +80,7 @@ class Permission(enum.Enum): MAKE_RELEASE = "MAKE_RELEASE" DELETE_RELEASE = "DELETE_RELEASE" ADD_SCREENSHOTS = "ADD_SCREENSHOTS" + REIMPORT_META = "REIMPORT_META" APPROVE_SCREENSHOT = "APPROVE_SCREENSHOT" APPROVE_RELEASE = "APPROVE_RELEASE" APPROVE_NEW = "APPROVE_NEW" @@ -358,11 +359,12 @@ class Dependency(db.Model): optional = db.Column(db.Boolean, nullable=False, default=False) __table_args__ = (db.UniqueConstraint("depender_id", "package_id", "meta_package_id", name="_dependency_uc"), ) - def __init__(self, depender=None, package=None, meta=None): + def __init__(self, depender=None, package=None, meta=None, optional=False): if depender is None: return self.depender = depender + self.optional = optional packageProvided = package is not None metaProvided = meta is not None @@ -673,6 +675,10 @@ class Package(db.Model): return url_for("packages.remove_self_maintainers", author=self.author.username, name=self.name) + def getUpdateFromReleaseURL(self): + return url_for("packages.update_from_release", + author=self.author.username, name=self.name) + def getReviewURL(self): return url_for('packages.review', author=self.author.username, name=self.name) @@ -705,7 +711,8 @@ class Package(db.Model): elif perm == Permission.MAKE_RELEASE or perm == Permission.ADD_SCREENSHOTS: return isMaintainer - elif perm == Permission.EDIT_PACKAGE or perm == Permission.APPROVE_CHANGES or perm == Permission.APPROVE_RELEASE: + elif perm == Permission.EDIT_PACKAGE or perm == Permission.REIMPORT_META or \ + perm == Permission.APPROVE_CHANGES or perm == Permission.APPROVE_RELEASE: return isMaintainer and user.rank.atLeast(UserRank.MEMBER if self.approved else UserRank.NEW_MEMBER) # Anyone can change the package name when not approved, but only editors when approved diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index 501b430..0568949 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -139,6 +139,60 @@ def cloneRepo(urlstr, ref=None, recursive=False): .replace("Cloning into '" + gitDir + "'...", "") \ .strip()) + +@celery.task(bind=True) +def updateMetaFromRelease(self, id, path): + release = PackageRelease.query.get(id) + if release is None: + raise TaskError("No such release!") + elif release.package is None: + raise TaskError("No package attached to release") + + temp = getTempDir() + try: + with ZipFile(path, 'r') as zip_ref: + zip_ref.extractall(temp) + + try: + tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \ + author=release.package.author.username, name=release.package.name) + + cache = {} + def getMetaPackages(names): + return [ MetaPackage.GetOrCreate(x, cache) for x in names ] + + provides = getMetaPackages(tree.fold("name")) + + package = release.package + package.provides.clear() + package.provides.extend(provides) + + for dep in package.dependencies: + if dep.meta_package: + db.session.delete(dep) + + for meta in getMetaPackages(tree.fold("meta", "depends")): + db.session.add(Dependency(package, meta=meta, optional=False)) + + for meta in getMetaPackages(tree.fold("meta", "optional_depends")): + db.session.add(Dependency(package, meta=meta, optional=True)) + + db.session.commit() + + except MinetestCheckError as err: + if "Fails validation" not in release.title: + release.title += " (Fails validation)" + + release.task_id = self.request.id + release.approved = False + db.session.commit() + + raise TaskError(str(err)) + + finally: + shutil.rmtree(temp) + + @celery.task() def getMeta(urlstr, author): gitDir, _ = cloneRepo(urlstr, recursive=True) @@ -249,6 +303,7 @@ def makeVCSRelease(id, branch): finally: shutil.rmtree(gitDir) + @celery.task() def importRepoScreenshot(id): package = Package.query.get(id) diff --git a/app/templates/admin/list.html b/app/templates/admin/list.html index 59dc8a0..26c7179 100644 --- a/app/templates/admin/list.html +++ b/app/templates/admin/list.html @@ -21,6 +21,7 @@ <select name="action"> <option value="delstuckreleases" selected>Delete stuck releases</option> <option value="checkreleases">Validate all Zip releases</option> + <option value="reimportpackages">Reimport meta</option> <option value="importmodlist">Import forum topics</option> <option value="recalcscores">Recalculate package scores</option> <option value="checkusers">Check forum users</option> diff --git a/app/templates/packages/create_edit.html b/app/templates/packages/create_edit.html index a293938..3e9277e 100644 --- a/app/templates/packages/create_edit.html +++ b/app/templates/packages/create_edit.html @@ -28,6 +28,17 @@ {{ _("Have you read the Package Inclusion Policy and Guidance yet?") }} </div> + {% if package and package.checkPerm(current_user, "REIMPORT_META") and package.releases.first() %} + <form class="alert alert-secondary mb-5" method="post" action="{{ package.getUpdateFromReleaseURL() }}"> + <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" /> + <input class="btn btn-sm btn-warning float-right" type="submit" value="{{ _('Import') }}" /> + + <b>{{ _("Reimport meta from latest release.") }}</b> + {{ _("This will override 'provides', 'dependencies', and 'optional_dependencies'.") }} + <div style="clear:both;"></div> + </form> + {% endif %} + <noscript> <div class="alert alert-warning"> {{ _("Javascript is needed to improve the user interface, and is needed for features |