From dbc56283ebb0d850fff65ecaf6753c0fa31e0c53 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Tue, 15 May 2018 19:35:59 +0100 Subject: Add screenshot importing from github --- app/tasks/importtasks.py | 48 +++++++++++++++-- app/utils.py | 132 +++++++++++++++++++++++++++++++++++++++++++++++ app/views/admin.py | 2 +- app/views/githublogin.py | 2 +- app/views/packages.py | 17 ++++-- app/views/tasks.py | 4 +- app/views/users.py | 2 +- app/views/utils.py | 132 ----------------------------------------------- 8 files changed, 194 insertions(+), 145 deletions(-) create mode 100644 app/utils.py delete mode 100644 app/views/utils.py diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index e0102b8..638e9ca 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -1,10 +1,12 @@ -import flask, json +import flask, json, os from flask.ext.sqlalchemy import SQLAlchemy +from urllib.error import HTTPError import urllib.request from urllib.parse import urlparse, quote_plus from app import app from app.models import * from app.tasks import celery, TaskError +from app.utils import randomString class GithubURLMaker: def __init__(self, url): @@ -37,7 +39,7 @@ class GithubURLMaker: return self.baseUrl + "/description.txt" def getScreenshotURL(self): - return self.baseUrl + "/placeholder.png" + return self.baseUrl + "/screenshot.png" def getCommitsURL(self, branch): return "https://api.github.com/repos/{}/{}/commits?sha={}" \ @@ -147,7 +149,7 @@ def getMeta(urlstr, author): result[key] = conf[key] except KeyError: pass - except OSError: + except HTTPError: print("mod.conf does not exist") if "name" in result: @@ -157,7 +159,7 @@ def getMeta(urlstr, author): try: contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8") result["description"] = contents.strip() - except OSError: + except HTTPError: print("description.txt does not exist!") if "description" in result: @@ -172,6 +174,7 @@ def getMeta(urlstr, author): return result + @celery.task() def makeVCSRelease(id, branch): release = PackageRelease.query.get(id) @@ -204,3 +207,40 @@ def makeVCSRelease(id, branch): db.session.commit() return release.url + + +@celery.task() +def importRepoScreenshot(id): + package = Package.query.get(id) + if package is None: + raise Exception("Unexpected none package") + + # Get URL Maker + url = urlparse(package.repo) + urlmaker = None + if url.netloc == "github.com": + urlmaker = GithubURLMaker(url) + else: + raise TaskError("Unsupported repo") + + if not urlmaker.isValid(): + raise TaskError("Error! Url maker not valid") + + try: + filename = randomString(10) + ".png" + imagePath = os.path.join(app.config["UPLOAD_FOLDER"], filename) + print(imagePath) + urllib.request.urlretrieve(urlmaker.getScreenshotURL(), imagePath) + + ss = PackageScreenshot() + ss.package = package + ss.title = "screenshot.png" + ss.url = "/uploads/" + filename + db.session.add(ss) + db.session.commit() + + return "/uploads/" + filename + except HTTPError: + print("screenshot.png does not exist") + + return None diff --git a/app/utils.py b/app/utils.py new file mode 100644 index 0000000..44647a1 --- /dev/null +++ b/app/utils.py @@ -0,0 +1,132 @@ +from flask import request, flash, abort, redirect +from flask_user import * +from flask_login import login_user, logout_user +from app.models import * +from app import app +import random, string, os + +def getExtension(filename): + return filename.rsplit(".", 1)[1].lower() if "." in filename else None + +def isFilenameAllowed(filename, exts): + return getExtension(filename) in exts + +def shouldReturnJson(): + return "application/json" in request.accept_mimetypes and \ + not "text/html" in request.accept_mimetypes + +def randomString(n): + return ''.join(random.choice(string.ascii_lowercase + \ + string.ascii_uppercase + string.digits) for _ in range(n)) + +def doFileUpload(file, allowedExtensions, fileTypeName): + if not file or file is None or file.filename == "": + flash("No selected file", "error") + return None + + ext = getExtension(file.filename) + if ext is None or not ext in allowedExtensions: + flash("Please upload load " + fileTypeName, "error") + return None + + filename = randomString(10) + "." + ext + file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) + return "/uploads/" + filename + + +def _do_login_user(user, remember_me=False): + def _call_or_get(v): + if callable(v): + return v() + else: + return v + + # User must have been authenticated + if not user: + return False + + user.active = True + if not user.rank.atLeast(UserRank.NEW_MEMBER): + user.rank = UserRank.NEW_MEMBER + + db.session.commit() + + # Check if user account has been disabled + if not _call_or_get(user.is_active): + flash("Your account has not been enabled.", "error") + return False + + # Check if user has a confirmed email address + user_manager = current_app.user_manager + if user_manager.enable_email and user_manager.enable_confirm_email \ + and not current_app.user_manager.enable_login_without_confirm_email \ + and not user.has_confirmed_email(): + url = url_for("user.resend_confirm_email") + flash("Your email address has not yet been confirmed", "error") + return False + + # Use Flask-Login to sign in user + login_user(user, remember=remember_me) + signals.user_logged_in.send(current_app._get_current_object(), user=user) + + flash("You have signed in successfully.", "success") + + return True + +def loginUser(user): + user_mixin = None + if user_manager.enable_username: + user_mixin = user_manager.find_user_by_username(user.username) + + return _do_login_user(user_mixin, False) + +def rank_required(rank): + def decorator(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if not current_user.is_authenticated: + return redirect(url_for("user.login")) + if not current_user.rank.atLeast(rank): + abort(403) + + return f(*args, **kwargs) + + return decorated_function + return decorator + +def getPackageByInfo(author, name): + user = User.query.filter_by(username=author).first() + if user is None: + abort(404) + + package = Package.query.filter_by(name=name, author_id=user.id).first() + if package is None: + abort(404) + + return package + +def is_package_page(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if not ("author" in kwargs and "name" in kwargs): + abort(400) + + package = getPackageByInfo(kwargs["author"], kwargs["name"]) + + del kwargs["author"] + del kwargs["name"] + + return f(package=package, *args, **kwargs) + + return decorated_function + +def triggerNotif(owner, causer, title, url): + if owner.rank.atLeast(UserRank.NEW_MEMBER) and owner != causer: + Notification.query.filter_by(user=owner, url=url).delete() + notif = Notification(owner, causer, title, url) + db.session.add(notif) + +def clearNotifications(url): + if current_user.is_authenticated: + Notification.query.filter_by(user=current_user, url=url).delete() + db.session.commit() diff --git a/app/views/admin.py b/app/views/admin.py index ee4d9f6..03dcc75 100644 --- a/app/views/admin.py +++ b/app/views/admin.py @@ -6,7 +6,7 @@ from app.models import * from app.tasks.forumtasks import importUsersFromModList from flask_wtf import FlaskForm from wtforms import * -from .utils import loginUser, rank_required +from app.utils import loginUser, rank_required @menu.register_menu(app, ".admin", "Admin", order=30, visible_when=lambda: current_user.rank.atLeast(UserRank.ADMIN)) diff --git a/app/views/githublogin.py b/app/views/githublogin.py index 114bb2b..27b197c 100644 --- a/app/views/githublogin.py +++ b/app/views/githublogin.py @@ -5,7 +5,7 @@ import flask_menu as menu from flask_github import GitHub from app import app, github from app.models import * -from .utils import loginUser +from app.utils import loginUser @app.route("/user/github/start/") def github_signin_page(): diff --git a/app/views/packages.py b/app/views/packages.py index 4a5fd3b..f2b17f1 100644 --- a/app/views/packages.py +++ b/app/views/packages.py @@ -3,10 +3,11 @@ from flask_user import * from flask.ext import menu from app import app from app.models import * -from app.tasks.importtasks import makeVCSRelease +from app.tasks.importtasks import importRepoScreenshot, makeVCSRelease -from .utils import * +from app.utils import * +from urllib.parse import urlparse from flask_wtf import FlaskForm from wtforms import * from wtforms.validators import * @@ -153,10 +154,11 @@ def create_edit_package_page(author=None, name=None): # Initial form class from post data and default data if request.method == "POST" and form.validate(): - # Successfully submitted! + wasNew = False if not package: package = Package() package.author = author + wasNew = True else: triggerNotif(package.author, current_user, "{} edited".format(package.title), package.getDetailsURL()) @@ -168,7 +170,14 @@ def create_edit_package_page(author=None, name=None): package.tags.append(Tag.query.get(tag)) db.session.commit() # save - return redirect(package.getDetailsURL()) # redirect + + if wasNew: + url = urlparse(package.repo) + if url.netloc == "github.com": + task = importRepoScreenshot.delay(package.id) + return redirect(url_for("check_task", id=task.id, r=package.getDetailsURL())) + + return redirect(package.getDetailsURL()) return render_template("packages/create_edit.html", package=package, form=form, author=author) diff --git a/app/views/tasks.py b/app/views/tasks.py index 3759543..150fb81 100644 --- a/app/views/tasks.py +++ b/app/views/tasks.py @@ -5,10 +5,10 @@ from app import app, csrf from app.models import * from app.tasks import celery, TaskError from app.tasks.importtasks import getMeta -from .utils import shouldReturnJson +from app.utils import shouldReturnJson # from celery.result import AsyncResult -from .utils import * +from app.utils import * @csrf.exempt @app.route("/tasks/getmeta/new/", methods=["POST"]) diff --git a/app/views/users.py b/app/views/users.py index ed195d4..19b0442 100644 --- a/app/views/users.py +++ b/app/views/users.py @@ -8,7 +8,7 @@ from flask_wtf import FlaskForm from flask_user.forms import RegisterForm from wtforms import * from wtforms.validators import * -from .utils import rank_required, randomString +from app.utils import rank_required, randomString from app.tasks.forumtasks import checkForumAccount from app.tasks.emails import sendVerifyEmail diff --git a/app/views/utils.py b/app/views/utils.py deleted file mode 100644 index 44647a1..0000000 --- a/app/views/utils.py +++ /dev/null @@ -1,132 +0,0 @@ -from flask import request, flash, abort, redirect -from flask_user import * -from flask_login import login_user, logout_user -from app.models import * -from app import app -import random, string, os - -def getExtension(filename): - return filename.rsplit(".", 1)[1].lower() if "." in filename else None - -def isFilenameAllowed(filename, exts): - return getExtension(filename) in exts - -def shouldReturnJson(): - return "application/json" in request.accept_mimetypes and \ - not "text/html" in request.accept_mimetypes - -def randomString(n): - return ''.join(random.choice(string.ascii_lowercase + \ - string.ascii_uppercase + string.digits) for _ in range(n)) - -def doFileUpload(file, allowedExtensions, fileTypeName): - if not file or file is None or file.filename == "": - flash("No selected file", "error") - return None - - ext = getExtension(file.filename) - if ext is None or not ext in allowedExtensions: - flash("Please upload load " + fileTypeName, "error") - return None - - filename = randomString(10) + "." + ext - file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename)) - return "/uploads/" + filename - - -def _do_login_user(user, remember_me=False): - def _call_or_get(v): - if callable(v): - return v() - else: - return v - - # User must have been authenticated - if not user: - return False - - user.active = True - if not user.rank.atLeast(UserRank.NEW_MEMBER): - user.rank = UserRank.NEW_MEMBER - - db.session.commit() - - # Check if user account has been disabled - if not _call_or_get(user.is_active): - flash("Your account has not been enabled.", "error") - return False - - # Check if user has a confirmed email address - user_manager = current_app.user_manager - if user_manager.enable_email and user_manager.enable_confirm_email \ - and not current_app.user_manager.enable_login_without_confirm_email \ - and not user.has_confirmed_email(): - url = url_for("user.resend_confirm_email") - flash("Your email address has not yet been confirmed", "error") - return False - - # Use Flask-Login to sign in user - login_user(user, remember=remember_me) - signals.user_logged_in.send(current_app._get_current_object(), user=user) - - flash("You have signed in successfully.", "success") - - return True - -def loginUser(user): - user_mixin = None - if user_manager.enable_username: - user_mixin = user_manager.find_user_by_username(user.username) - - return _do_login_user(user_mixin, False) - -def rank_required(rank): - def decorator(f): - @wraps(f) - def decorated_function(*args, **kwargs): - if not current_user.is_authenticated: - return redirect(url_for("user.login")) - if not current_user.rank.atLeast(rank): - abort(403) - - return f(*args, **kwargs) - - return decorated_function - return decorator - -def getPackageByInfo(author, name): - user = User.query.filter_by(username=author).first() - if user is None: - abort(404) - - package = Package.query.filter_by(name=name, author_id=user.id).first() - if package is None: - abort(404) - - return package - -def is_package_page(f): - @wraps(f) - def decorated_function(*args, **kwargs): - if not ("author" in kwargs and "name" in kwargs): - abort(400) - - package = getPackageByInfo(kwargs["author"], kwargs["name"]) - - del kwargs["author"] - del kwargs["name"] - - return f(package=package, *args, **kwargs) - - return decorated_function - -def triggerNotif(owner, causer, title, url): - if owner.rank.atLeast(UserRank.NEW_MEMBER) and owner != causer: - Notification.query.filter_by(user=owner, url=url).delete() - notif = Notification(owner, causer, title, url) - db.session.add(notif) - -def clearNotifications(url): - if current_user.is_authenticated: - Notification.query.filter_by(user=current_user, url=url).delete() - db.session.commit() -- cgit v1.2.3